get rid of upload service

Signed-off-by: Andrey Platov <andrey@hardcoreeng.com>

common/config/rush/pnpm-lock.yaml

@@ -112,6 +112,7 @@ specifiers:
   express: ^4.17.1
   express-fileupload: ^1.2.1
   file-loader: ^6.2.0
+  filesize: ^8.0.3
   intl-messageformat: ^9.7.1
   just-clone: ^3.2.1
   koa: ^2.13.1
@@ -250,6 +251,7 @@ dependencies:
   express: 4.17.1
   express-fileupload: 1.2.1
   file-loader: 6.2.0_webpack@5.57.1
+  filesize: 8.0.3
   intl-messageformat: 9.7.1
   just-clone: 3.2.1
   koa: 2.13.3
@@ -9319,7 +9321,7 @@ packages:
     dev: false
 
   file:projects/chunter-resources.tgz_e1367da94684b005adf08f025c517b1a:
-    resolution: {integrity: sha512-ffIxP0yesLoUxgDnGwHBhD0MRgzq5REyfoon10k7s3nlwDJzC3/yjPgMdFc9p+rZUfcRdhA17vF+AkuOD5vycA==, tarball: file:projects/chunter-resources.tgz}
+    resolution: {integrity: sha512-zq2XzUCiMo5EZ7KfbLCGceDlQIyplRUNg8hPJP/7npIY/3YTgWDoc8svgCPltsen4PNMioIjAWWP1t+g62i6Mg==, tarball: file:projects/chunter-resources.tgz}
     id: file:projects/chunter-resources.tgz
     name: '@rush-temp/chunter-resources'
     version: 0.0.0
@@ -9608,21 +9610,30 @@ packages:
     dev: false
 
   file:projects/front.tgz_b4fae2aaf9a34e02c9acb1cfc4c88710:
-    resolution: {integrity: sha512-XmgU22kXGEisU6bW4q3GuuqhzjWazpokQ+ASxb2Nu7M/cTM+xy9EY/fvessk79EXRTGFvoH4hM2jrbleWKP9ug==, tarball: file:projects/front.tgz}
+    resolution: {integrity: sha512-qKmnx2yFt1hFJRZWJNaPPbb+p/Q0lnyA6u9AAgeqDbOjmWjp48NxqX6jIbiEyQi56IDHViE3w4/NQb0c/mp/rA==, tarball: file:projects/front.tgz}
     id: file:projects/front.tgz
     name: '@rush-temp/front'
     version: 0.0.0
     dependencies:
+      '@types/cors': 2.8.12
       '@types/express': 4.17.13
+      '@types/express-fileupload': 1.1.7
       '@types/heft-jest': 1.0.2
+      '@types/minio': 7.0.10
       '@types/node': 16.10.3
+      '@types/uuid': 8.3.1
       '@typescript-eslint/eslint-plugin': 4.33.0_d753869925cce96d3eb2141eeedafe57
+      cors: 2.8.5
       esbuild: 0.12.29
       eslint: 7.32.0
       eslint-plugin-import: 2.24.2_eslint@7.32.0
       eslint-plugin-node: 11.1.0_eslint@7.32.0
       eslint-plugin-promise: 4.3.1
       express: 4.17.1
+      express-fileupload: 1.2.1
+      jwt-simple: 0.5.6
+      minio: 7.0.19
+      uuid: 8.3.2
     transitivePeerDependencies:
       - '@typescript-eslint/parser'
       - supports-color

dev/prod/.env-prod

@@ -2,4 +2,4 @@
 
 #ACCOUNTS_URL=https://ftwm71rwag.execute-api.us-west-2.amazonaws.com/stage/
 ACCOUNTS_URL=https://account.hc.engineering/
-UPLOAD_URL=https://upload.hc.engineering/
+UPLOAD_URL=/files

dev/prod/webpack.config.js

@@ -171,7 +171,7 @@ module.exports = {
       '/upload': {
         // target: 'https://anticrm-upload.herokuapp.com/',
         // target: 'http://localhost:3000/',
-        target: 'https://upload.hc.engineering/',
+        target: 'https://front.hc.engineering/files',
         changeOrigin: true,
         pathRewrite: { '^/upload': '' },
         logLevel: 'debug'

server/front/kube/front.yml

@@ -19,6 +19,29 @@ spec:
           ports:
             - containerPort: 8080
           imagePullPolicy: Always
+          env:
+          - name: TRANSACTOR_URL
+            value: ws://transactor/
+          - name: ELASTIC_URL
+            valueFrom:
+              secretKeyRef:
+                name: elastic
+                key: url
+          - name: MINIO_ENDPOINT
+            valueFrom:
+              secretKeyRef:
+                name: minio
+                key: endpoint
+          - name: MINIO_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: minio
+                key: accessKey
+          - name: MINIO_SECRET_KEY
+            valueFrom:
+              secretKeyRef:
+                name: minio
+                key: secretKey
 ---
 apiVersion: v1
 kind: Service

server/front/package.json

@@ -7,7 +7,7 @@
   "scripts": {
     "build": "heft build",
     "lint:fix": "eslint --fix src",
-    "bundle": "esbuild src/index.ts --bundle --minify --platform=node > bundle.js & rm -rf ./dist && cp -r ../../dev/prod/dist . && cp -r ../../dev/prod/public/* ./dist/",
+    "bundle": "esbuild src/__start.ts --bundle --minify --platform=node > bundle.js & rm -rf ./dist && cp -r ../../dev/prod/dist . && cp -r ../../dev/prod/public/* ./dist/",
     "docker:build": "docker build -t anticrm/front .",
     "docker:push": "docker push anticrm/front"
   },
@@ -21,11 +21,24 @@
     "eslint-plugin-node":"11",
     "eslint":"^7.32.0",
     "@types/express":"^4.17.13",
+    "@types/express-fileupload":"^1.1.7",
+    "@types/uuid":"^8.3.1",
+    "@types/cors":"^2.8.12",
+    "@types/minio":"^7.0.10",
     "esbuild":"^0.12.24"
   },
   "dependencies": {
     "@anticrm/core": "~0.6.11",
     "@anticrm/platform": "~0.6.5",
-    "express": "^4.17.1"
+    "express": "^4.17.1",
+    "express-fileupload":"^1.2.1",
+    "uuid":"^8.3.2",
+    "cors":"^2.8.5",
+    "@anticrm/elastic":"~0.6.0",
+    "jwt-simple":"^0.5.6",
+    "@anticrm/server-core":"~0.6.1",
+    "@anticrm/chunter":"~0.6.0",
+    "@anticrm/contrib":"~0.6.0",
+    "minio":"^7.0.19"
   }
 }

server/front/src/__start.ts

@@ -0,0 +1,58 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { start } from './app'
+import { Client } from 'minio'
+
+const url = process.env.TRANSACTOR_URL
+if (url === undefined) {
+  console.error('please provide transactor url')
+  process.exit(1)
+}
+
+const elasticUrl = process.env.ELASTIC_URL
+if (elasticUrl === undefined) {
+  console.error('please provide elastic url')
+  process.exit(1)
+}
+
+const minioEndpoint = process.env.MINIO_ENDPOINT
+if (minioEndpoint === undefined) {
+  console.error('please provide minio endpoint')
+  process.exit(1)
+}
+
+const minioAccessKey = process.env.MINIO_ACCESS_KEY
+if (minioAccessKey === undefined) {
+  console.error('please provide minio access key')
+  process.exit(1)
+}
+
+const minioSecretKey = process.env.MINIO_SECRET_KEY
+if (minioSecretKey === undefined) {
+  console.error('please provide minio secret key')
+  process.exit(1)
+}
+
+const minio = new Client({
+  endPoint: minioEndpoint,
+  port: 9000,
+  useSSL: false,
+  accessKey: minioAccessKey,
+  secretKey: minioSecretKey
+})
+
+start(url, elasticUrl, minio, 8080)

server/front/src/app.ts

@@ -0,0 +1,190 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { resolve, join } from 'path'
+import express from 'express'
+import fileUpload, { UploadedFile } from 'express-fileupload'
+import cors from 'cors'
+import { v4 as uuid } from 'uuid'
+import { decode } from 'jwt-simple'
+
+import { Space, Ref, Doc, Account, generateId } from '@anticrm/core'
+// import { TxFactory } from '@anticrm/core'
+import type { Token, IndexedDoc } from '@anticrm/server-core'
+import { createElasticAdapter } from '@anticrm/elastic'
+import chunter from '@anticrm/chunter'
+// import { createContributingClient } from '@anticrm/contrib'
+
+import { Client, ItemBucketMetadata } from 'minio'
+
+// import { createElasticAdapter } from '@anticrm/elastic'
+
+// const BUCKET = 'anticrm-upload-9e4e89c'
+
+// async function awsUpload (file: UploadedFile): Promise<string> {
+//   const id = uuid()
+//   const s3 = new S3()
+//   const resp = await s3.upload({
+//     Bucket: BUCKET,
+//     Key: id,
+//     Body: file.data,
+//     ContentType: file.mimetype,
+//     ACL: 'public-read'
+//   }).promise()
+//   console.log(resp)
+//   return id
+// }
+
+async function minioUpload (minio: Client, workspace: string, file: UploadedFile): Promise<string> {
+  const id = uuid()
+  const meta: ItemBucketMetadata = {
+    'Content-Type': file.mimetype
+  }
+
+  const resp = await minio.putObject(workspace, id, file.data, file.size, meta)
+
+  console.log(resp)
+  return id
+}
+
+// async function createAttachment (endpoint: string, token: string, account: Ref<Account>, space: Ref<Space>, attachedTo: Ref<Doc>, collection: string, name: string, file: string): Promise<void> {
+//   const txFactory = new TxFactory(account)
+//   const tx = txFactory.createTxCreateDoc(chunter.class.Attachment, space, {
+//     attachedTo,
+//     collection,
+//     name,
+//     file
+//   })
+//   const url = new URL(`/${token}`, endpoint)
+//   const client = await createContributingClient(url.href)
+//   await client.tx(tx)
+//   client.close()
+// }
+
+/**
+ * @public
+ * @param port -
+ */
+export function start (transactorEndpoint: string, elasticUrl: string, minio: Client, port: number): void {
+  const app = express()
+
+  app.use(cors())
+  app.use(fileUpload())
+
+  const dist = resolve(__dirname, 'dist')
+  console.log('serving static files from', dist)
+  app.use(express.static(dist, { maxAge: '10m' }))
+
+  // eslint-disable-next-line @typescript-eslint/no-misused-promises
+  app.get('/files', async (req, res) => {
+    try {
+      const token = req.query.token as string
+      const payload = decode(token, 'secret', false) as Token
+      const uuid = req.query.file as string
+
+      const stat = await minio.statObject(payload.workspace, uuid)
+      minio.getObject(payload.workspace, uuid, function (err, dataStream) {
+        if (err !== null) {
+          return console.log(err)
+        }
+        res.status(200)
+
+        const contentType = stat.metaData['content-type']
+        if (contentType !== undefined) {
+          res.setHeader('Content-Type', contentType)
+        }
+
+        dataStream.on('data', function (chunk) {
+          res.write(chunk)
+        })
+        dataStream.on('end', function () {
+          res.end()
+        })
+        dataStream.on('error', function (err) {
+          console.log(err)
+        })
+      })
+    } catch (error) {
+      console.log(error)
+      res.status(500).send()
+    }
+  })
+
+  // eslint-disable-next-line @typescript-eslint/no-misused-promises
+  app.post('/files', async (req, res) => {
+    const file = req.files?.file as UploadedFile
+
+    if (file === undefined) {
+      res.status(400).send()
+      return
+    }
+
+    const authHeader = req.headers.authorization
+    if (authHeader === undefined) {
+      res.status(403).send()
+      return
+    }
+
+    try {
+      const token = authHeader.split(' ')[1]
+      const payload = decode(token ?? '', 'secret', false) as Token
+      // const fileId = await awsUpload(file as UploadedFile)
+      const uuid = await minioUpload(minio, payload.workspace, file)
+      console.log('uploaded uuid', uuid)
+
+      const name = req.query.name as string
+      const space = req.query.space as Ref<Space>
+      const attachedTo = req.query.attachedTo as Ref<Doc>
+      // const name = req.query.name as string
+
+      // await createAttachment(
+      //   transactorEndpoint,
+      //   token,
+      //   'core:account:System' as Ref<Account>,
+      //   space,
+      //   attachedTo,
+      //   collection,
+      //   name,
+      //   fileId
+      // )
+
+      const elastic = await createElasticAdapter(elasticUrl, payload.workspace)
+
+      const indexedDoc: IndexedDoc = {
+        id: generateId() + '/attachments/' + name,
+        _class: chunter.class.Attachment,
+        space,
+        modifiedOn: Date.now(),
+        modifiedBy: 'core:account:System' as Ref<Account>,
+        attachedTo,
+        data: file.data.toString('base64')
+      }
+
+      await elastic.index(indexedDoc)
+
+      res.status(200).send(uuid)
+    } catch (error) {
+      console.log(error)
+      res.status(500).send()
+    }
+  })
+
+  app.get('*', function (request, response) {
+    response.sendFile(join(dist, 'index.html'))
+  })
+
+  app.listen(port)
+}