From 8fd1adfe0a11ba0b2ff6c0e6bd15428746a5773f Mon Sep 17 00:00:00 2001
From: Andrey Platov <andrey@hardcoreeng.com>
Date: Mon, 25 Oct 2021 17:11:48 +0200
Subject: [PATCH] get rid of `upload` service
Signed-off-by: Andrey Platov <andrey@hardcoreeng.com>
---
common/config/rush/pnpm-lock.yaml | 15 ++-
dev/prod/.env-prod | 2 +-
dev/prod/webpack.config.js | 2 +-
server/front/kube/front.yml | 23 ++++
server/front/package.json | 17 ++-
server/front/src/__start.ts | 58 +++++++++
server/front/src/app.ts | 190 ++++++++++++++++++++++++++++++
7 files changed, 301 insertions(+), 6 deletions(-)
create mode 100644 server/front/src/__start.ts
create mode 100644 server/front/src/app.ts
diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml
index 4b0642a81f..c28de2e19f 100644
--- a/common/config/rush/pnpm-lock.yaml
+++ b/common/config/rush/pnpm-lock.yaml
@@ -112,6 +112,7 @@ specifiers:
express: ^4.17.1
express-fileupload: ^1.2.1
file-loader: ^6.2.0
+ filesize: ^8.0.3
intl-messageformat: ^9.7.1
just-clone: ^3.2.1
koa: ^2.13.1
@@ -250,6 +251,7 @@ dependencies:
express: 4.17.1
express-fileupload: 1.2.1
file-loader: 6.2.0_webpack@5.57.1
+ filesize: 8.0.3
intl-messageformat: 9.7.1
just-clone: 3.2.1
koa: 2.13.3
@@ -9319,7 +9321,7 @@ packages:
dev: false
file:projects/chunter-resources.tgz_e1367da94684b005adf08f025c517b1a:
- resolution: {integrity: sha512-ffIxP0yesLoUxgDnGwHBhD0MRgzq5REyfoon10k7s3nlwDJzC3/yjPgMdFc9p+rZUfcRdhA17vF+AkuOD5vycA==, tarball: file:projects/chunter-resources.tgz}
+ resolution: {integrity: sha512-zq2XzUCiMo5EZ7KfbLCGceDlQIyplRUNg8hPJP/7npIY/3YTgWDoc8svgCPltsen4PNMioIjAWWP1t+g62i6Mg==, tarball: file:projects/chunter-resources.tgz}
id: file:projects/chunter-resources.tgz
name: '@rush-temp/chunter-resources'
version: 0.0.0
@@ -9608,21 +9610,30 @@ packages:
dev: false
file:projects/front.tgz_b4fae2aaf9a34e02c9acb1cfc4c88710:
- resolution: {integrity: sha512-XmgU22kXGEisU6bW4q3GuuqhzjWazpokQ+ASxb2Nu7M/cTM+xy9EY/fvessk79EXRTGFvoH4hM2jrbleWKP9ug==, tarball: file:projects/front.tgz}
+ resolution: {integrity: sha512-qKmnx2yFt1hFJRZWJNaPPbb+p/Q0lnyA6u9AAgeqDbOjmWjp48NxqX6jIbiEyQi56IDHViE3w4/NQb0c/mp/rA==, tarball: file:projects/front.tgz}
id: file:projects/front.tgz
name: '@rush-temp/front'
version: 0.0.0
dependencies:
+ '@types/cors': 2.8.12
'@types/express': 4.17.13
+ '@types/express-fileupload': 1.1.7
'@types/heft-jest': 1.0.2
+ '@types/minio': 7.0.10
'@types/node': 16.10.3
+ '@types/uuid': 8.3.1
'@typescript-eslint/eslint-plugin': 4.33.0_d753869925cce96d3eb2141eeedafe57
+ cors: 2.8.5
esbuild: 0.12.29
eslint: 7.32.0
eslint-plugin-import: 2.24.2_eslint@7.32.0
eslint-plugin-node: 11.1.0_eslint@7.32.0
eslint-plugin-promise: 4.3.1
express: 4.17.1
+ express-fileupload: 1.2.1
+ jwt-simple: 0.5.6
+ minio: 7.0.19
+ uuid: 8.3.2
transitivePeerDependencies:
- '@typescript-eslint/parser'
- supports-color
diff --git a/dev/prod/.env-prod b/dev/prod/.env-prod
index 1188498b80..f169c875ee 100644
--- a/dev/prod/.env-prod
+++ b/dev/prod/.env-prod
@@ -2,4 +2,4 @@
#ACCOUNTS_URL=https://ftwm71rwag.execute-api.us-west-2.amazonaws.com/stage/
ACCOUNTS_URL=https://account.hc.engineering/
-UPLOAD_URL=https://upload.hc.engineering/
+UPLOAD_URL=/files
diff --git a/dev/prod/webpack.config.js b/dev/prod/webpack.config.js
index 0b8c1bce98..808e0945c9 100644
--- a/dev/prod/webpack.config.js
+++ b/dev/prod/webpack.config.js
@@ -171,7 +171,7 @@ module.exports = {
'/upload': {
// target: 'https://anticrm-upload.herokuapp.com/',
// target: 'http://localhost:3000/',
- target: 'https://upload.hc.engineering/',
+ target: 'https://front.hc.engineering/files',
changeOrigin: true,
pathRewrite: { '^/upload': '' },
logLevel: 'debug'
diff --git a/server/front/kube/front.yml b/server/front/kube/front.yml
index 1e22c08c51..0eb6c3eadf 100644
--- a/server/front/kube/front.yml
+++ b/server/front/kube/front.yml
@@ -19,6 +19,29 @@ spec:
ports:
- containerPort: 8080
imagePullPolicy: Always
+ env:
+ - name: TRANSACTOR_URL
+ value: ws://transactor/
+ - name: ELASTIC_URL
+ valueFrom:
+ secretKeyRef:
+ name: elastic
+ key: url
+ - name: MINIO_ENDPOINT
+ valueFrom:
+ secretKeyRef:
+ name: minio
+ key: endpoint
+ - name: MINIO_ACCESS_KEY
+ valueFrom:
+ secretKeyRef:
+ name: minio
+ key: accessKey
+ - name: MINIO_SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ name: minio
+ key: secretKey
---
apiVersion: v1
kind: Service
diff --git a/server/front/package.json b/server/front/package.json
index cd2e3a7599..96ac4e010c 100644
--- a/server/front/package.json
+++ b/server/front/package.json
@@ -7,7 +7,7 @@
"scripts": {
"build": "heft build",
"lint:fix": "eslint --fix src",
- "bundle": "esbuild src/index.ts --bundle --minify --platform=node > bundle.js & rm -rf ./dist && cp -r ../../dev/prod/dist . && cp -r ../../dev/prod/public/* ./dist/",
+ "bundle": "esbuild src/__start.ts --bundle --minify --platform=node > bundle.js & rm -rf ./dist && cp -r ../../dev/prod/dist . && cp -r ../../dev/prod/public/* ./dist/",
"docker:build": "docker build -t anticrm/front .",
"docker:push": "docker push anticrm/front"
},
@@ -21,11 +21,24 @@
"eslint-plugin-node":"11",
"eslint":"^7.32.0",
"@types/express":"^4.17.13",
+ "@types/express-fileupload":"^1.1.7",
+ "@types/uuid":"^8.3.1",
+ "@types/cors":"^2.8.12",
+ "@types/minio":"^7.0.10",
"esbuild":"^0.12.24"
},
"dependencies": {
"@anticrm/core": "~0.6.11",
"@anticrm/platform": "~0.6.5",
- "express": "^4.17.1"
+ "express": "^4.17.1",
+ "express-fileupload":"^1.2.1",
+ "uuid":"^8.3.2",
+ "cors":"^2.8.5",
+ "@anticrm/elastic":"~0.6.0",
+ "jwt-simple":"^0.5.6",
+ "@anticrm/server-core":"~0.6.1",
+ "@anticrm/chunter":"~0.6.0",
+ "@anticrm/contrib":"~0.6.0",
+ "minio":"^7.0.19"
}
}
diff --git a/server/front/src/__start.ts b/server/front/src/__start.ts
new file mode 100644
index 0000000000..7f9dacb089
--- /dev/null
+++ b/server/front/src/__start.ts
@@ -0,0 +1,58 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { start } from './app'
+import { Client } from 'minio'
+
+const url = process.env.TRANSACTOR_URL
+if (url === undefined) {
+ console.error('please provide transactor url')
+ process.exit(1)
+}
+
+const elasticUrl = process.env.ELASTIC_URL
+if (elasticUrl === undefined) {
+ console.error('please provide elastic url')
+ process.exit(1)
+}
+
+const minioEndpoint = process.env.MINIO_ENDPOINT
+if (minioEndpoint === undefined) {
+ console.error('please provide minio endpoint')
+ process.exit(1)
+}
+
+const minioAccessKey = process.env.MINIO_ACCESS_KEY
+if (minioAccessKey === undefined) {
+ console.error('please provide minio access key')
+ process.exit(1)
+}
+
+const minioSecretKey = process.env.MINIO_SECRET_KEY
+if (minioSecretKey === undefined) {
+ console.error('please provide minio secret key')
+ process.exit(1)
+}
+
+const minio = new Client({
+ endPoint: minioEndpoint,
+ port: 9000,
+ useSSL: false,
+ accessKey: minioAccessKey,
+ secretKey: minioSecretKey
+})
+
+start(url, elasticUrl, minio, 8080)
diff --git a/server/front/src/app.ts b/server/front/src/app.ts
new file mode 100644
index 0000000000..cbf267b6f5
--- /dev/null
+++ b/server/front/src/app.ts
@@ -0,0 +1,190 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { resolve, join } from 'path'
+import express from 'express'
+import fileUpload, { UploadedFile } from 'express-fileupload'
+import cors from 'cors'
+import { v4 as uuid } from 'uuid'
+import { decode } from 'jwt-simple'
+
+import { Space, Ref, Doc, Account, generateId } from '@anticrm/core'
+// import { TxFactory } from '@anticrm/core'
+import type { Token, IndexedDoc } from '@anticrm/server-core'
+import { createElasticAdapter } from '@anticrm/elastic'
+import chunter from '@anticrm/chunter'
+// import { createContributingClient } from '@anticrm/contrib'
+
+import { Client, ItemBucketMetadata } from 'minio'
+
+// import { createElasticAdapter } from '@anticrm/elastic'
+
+// const BUCKET = 'anticrm-upload-9e4e89c'
+
+// async function awsUpload (file: UploadedFile): Promise<string> {
+// const id = uuid()
+// const s3 = new S3()
+// const resp = await s3.upload({
+// Bucket: BUCKET,
+// Key: id,
+// Body: file.data,
+// ContentType: file.mimetype,
+// ACL: 'public-read'
+// }).promise()
+// console.log(resp)
+// return id
+// }
+
+async function minioUpload (minio: Client, workspace: string, file: UploadedFile): Promise<string> {
+ const id = uuid()
+ const meta: ItemBucketMetadata = {
+ 'Content-Type': file.mimetype
+ }
+
+ const resp = await minio.putObject(workspace, id, file.data, file.size, meta)
+
+ console.log(resp)
+ return id
+}
+
+// async function createAttachment (endpoint: string, token: string, account: Ref<Account>, space: Ref<Space>, attachedTo: Ref<Doc>, collection: string, name: string, file: string): Promise<void> {
+// const txFactory = new TxFactory(account)
+// const tx = txFactory.createTxCreateDoc(chunter.class.Attachment, space, {
+// attachedTo,
+// collection,
+// name,
+// file
+// })
+// const url = new URL(`/${token}`, endpoint)
+// const client = await createContributingClient(url.href)
+// await client.tx(tx)
+// client.close()
+// }
+
+/**
+ * @public
+ * @param port -
+ */
+export function start (transactorEndpoint: string, elasticUrl: string, minio: Client, port: number): void {
+ const app = express()
+
+ app.use(cors())
+ app.use(fileUpload())
+
+ const dist = resolve(__dirname, 'dist')
+ console.log('serving static files from', dist)
+ app.use(express.static(dist, { maxAge: '10m' }))
+
+ // eslint-disable-next-line @typescript-eslint/no-misused-promises
+ app.get('/files', async (req, res) => {
+ try {
+ const token = req.query.token as string
+ const payload = decode(token, 'secret', false) as Token
+ const uuid = req.query.file as string
+
+ const stat = await minio.statObject(payload.workspace, uuid)
+ minio.getObject(payload.workspace, uuid, function (err, dataStream) {
+ if (err !== null) {
+ return console.log(err)
+ }
+ res.status(200)
+
+ const contentType = stat.metaData['content-type']
+ if (contentType !== undefined) {
+ res.setHeader('Content-Type', contentType)
+ }
+
+ dataStream.on('data', function (chunk) {
+ res.write(chunk)
+ })
+ dataStream.on('end', function () {
+ res.end()
+ })
+ dataStream.on('error', function (err) {
+ console.log(err)
+ })
+ })
+ } catch (error) {
+ console.log(error)
+ res.status(500).send()
+ }
+ })
+
+ // eslint-disable-next-line @typescript-eslint/no-misused-promises
+ app.post('/files', async (req, res) => {
+ const file = req.files?.file as UploadedFile
+
+ if (file === undefined) {
+ res.status(400).send()
+ return
+ }
+
+ const authHeader = req.headers.authorization
+ if (authHeader === undefined) {
+ res.status(403).send()
+ return
+ }
+
+ try {
+ const token = authHeader.split(' ')[1]
+ const payload = decode(token ?? '', 'secret', false) as Token
+ // const fileId = await awsUpload(file as UploadedFile)
+ const uuid = await minioUpload(minio, payload.workspace, file)
+ console.log('uploaded uuid', uuid)
+
+ const name = req.query.name as string
+ const space = req.query.space as Ref<Space>
+ const attachedTo = req.query.attachedTo as Ref<Doc>
+ // const name = req.query.name as string
+
+ // await createAttachment(
+ // transactorEndpoint,
+ // token,
+ // 'core:account:System' as Ref<Account>,
+ // space,
+ // attachedTo,
+ // collection,
+ // name,
+ // fileId
+ // )
+
+ const elastic = await createElasticAdapter(elasticUrl, payload.workspace)
+
+ const indexedDoc: IndexedDoc = {
+ id: generateId() + '/attachments/' + name,
+ _class: chunter.class.Attachment,
+ space,
+ modifiedOn: Date.now(),
+ modifiedBy: 'core:account:System' as Ref<Account>,
+ attachedTo,
+ data: file.data.toString('base64')
+ }
+
+ await elastic.index(indexedDoc)
+
+ res.status(200).send(uuid)
+ } catch (error) {
+ console.log(error)
+ res.status(500).send()
+ }
+ })
+
+ app.get('*', function (request, response) {
+ response.sendFile(join(dist, 'index.html'))
+ })
+
+ app.listen(port)
+}