From 8b92648adec2f322e5e836d0014ec44ff484214a Mon Sep 17 00:00:00 2001 From: Andrey Platov Date: Sun, 29 Aug 2021 17:30:22 +0200 Subject: [PATCH] upload rework Signed-off-by: Andrey Platov --- common/config/rush/pnpm-lock.yaml | 47 ++--------- deploy/setup-es-attachment-pipeline.sh | 29 +++++++ images/Dockerfile.elastic | 2 + images/build-elastic.sh | 18 +++++ packages/core/src/tx.ts | 5 +- .../src/components/CreateCandidate.svelte | 2 +- .../src/components/DialogHeader.svelte | 44 ++++++----- plugins/recruit-resources/src/utils.ts | 40 ++++++++++ server/core/src/fulltext.ts | 4 +- server/core/src/types.ts | 13 ++++ server/elastic/src/__tests__/adapter.test.ts | 6 +- server/upload/build.sh | 20 +++++ server/upload/package.json | 2 - server/upload/src/app.ts | 78 +++++++++++-------- 14 files changed, 208 insertions(+), 102 deletions(-) create mode 100755 deploy/setup-es-attachment-pipeline.sh create mode 100644 images/Dockerfile.elastic create mode 100755 images/build-elastic.sh create mode 100644 plugins/recruit-resources/src/utils.ts create mode 100755 server/upload/build.sh diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index fd52183e31..25935c41d6 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -8202,37 +8202,6 @@ packages: yn: 3.1.1 dev: false - /ts-node/10.2.1_f50b86b1778cd2aa4e5405c08bb39559: - resolution: {integrity: sha512-hCnyOyuGmD5wHleOQX6NIjJtYVIO8bPP8F2acWkB4W06wdlkgyvJtubO/I9NkI88hCFECbsEgoLc0VNkYmcSfw==} - engines: {node: '>=12.0.0'} - hasBin: true - peerDependencies: - '@swc/core': '>=1.2.50' - '@swc/wasm': '>=1.2.50' - '@types/node': '*' - typescript: '>=2.7' - peerDependenciesMeta: - '@swc/core': - optional: true - '@swc/wasm': - optional: true - dependencies: - '@cspotcode/source-map-support': 0.6.1 - '@tsconfig/node10': 1.0.8 - '@tsconfig/node12': 1.0.9 - '@tsconfig/node14': 1.0.1 - '@tsconfig/node16': 1.0.2 - '@types/node': 16.7.1 - acorn: 8.4.1 - acorn-walk: 8.1.1 - arg: 4.1.3 - create-require: 1.1.1 - diff: 4.0.2 - make-error: 1.3.6 - typescript: 4.3.5 - yn: 3.1.1 - dev: false - /ts-node/10.2.1_typescript@4.3.5: resolution: {integrity: sha512-hCnyOyuGmD5wHleOQX6NIjJtYVIO8bPP8F2acWkB4W06wdlkgyvJtubO/I9NkI88hCFECbsEgoLc0VNkYmcSfw==} engines: {node: '>=12.0.0'} @@ -9164,7 +9133,7 @@ packages: version: 0.0.0 dependencies: '@types/heft-jest': 1.0.2 - '@types/node': 16.7.1 + '@types/node': 16.7.5 '@types/ws': 7.4.7 '@typescript-eslint/eslint-plugin': 4.28.5_a8e83fcad666e1ba86be4b2e27a20aea eslint: 7.32.0 @@ -9298,23 +9267,23 @@ packages: dev: false file:projects/elastic.tgz_6c259fadfeb3a4b20890aefe87070b8b: - resolution: {integrity: sha512-uoPGQmOAopNC8F64rYWrRDU1s3/u70CNQeBYz7EH/PE6mR0QsYIFJW3U6/LgeDWsMXNge+5fKkjwAJh0+EQtBA==, tarball: file:projects/elastic.tgz} + resolution: {integrity: sha512-IEQdOVDEl4SLQUo1mu4wppfdnbAyiAMp38TL1yfF+Xh7J99/R7uSqQq0pI+nsUXKAR0oSsmMoHPiEYo7Kc15Ig==, tarball: file:projects/elastic.tgz} id: file:projects/elastic.tgz name: '@rush-temp/elastic' version: 0.0.0 dependencies: '@elastic/elasticsearch': 7.14.0 '@types/heft-jest': 1.0.2 + '@types/node': 16.7.5 '@typescript-eslint/eslint-plugin': 4.28.5_a8e83fcad666e1ba86be4b2e27a20aea eslint: 7.32.0 eslint-plugin-import: 2.23.4_eslint@7.32.0 eslint-plugin-node: 11.1.0_eslint@7.32.0 eslint-plugin-promise: 4.3.1 - ts-node: 10.2.1_typescript@4.3.5 + ts-node: 10.2.1_eb14afb1492fcd444e277f1fdb668e87 transitivePeerDependencies: - '@swc/core' - '@swc/wasm' - - '@types/node' - '@typescript-eslint/parser' - supports-color - typescript @@ -9328,7 +9297,7 @@ packages: dependencies: '@types/express': 4.17.13 '@types/heft-jest': 1.0.2 - '@types/node': 16.7.1 + '@types/node': 16.7.5 '@typescript-eslint/eslint-plugin': 4.28.5_a8e83fcad666e1ba86be4b2e27a20aea esbuild: 0.12.24 eslint: 7.32.0 @@ -9395,13 +9364,13 @@ packages: name: '@rush-temp/model-all' version: 0.0.0 dependencies: - '@types/node': 16.7.1 + '@types/node': 16.7.5 '@typescript-eslint/eslint-plugin': 4.28.5_a8e83fcad666e1ba86be4b2e27a20aea eslint: 7.32.0 eslint-plugin-import: 2.23.4_eslint@7.32.0 eslint-plugin-node: 11.1.0_eslint@7.32.0 eslint-plugin-promise: 4.3.1 - ts-node: 10.2.1_f50b86b1778cd2aa4e5405c08bb39559 + ts-node: 10.2.1_eb14afb1492fcd444e277f1fdb668e87 transitivePeerDependencies: - '@swc/core' - '@swc/wasm' @@ -10167,7 +10136,7 @@ packages: dev: false file:projects/workspace.tgz_6c259fadfeb3a4b20890aefe87070b8b: - resolution: {integrity: sha512-xsi1Oqd00ZntGoGAMsxj9vzdN9ff9tEgf//ZmaAH4GqaVT0VF2pzeWwDNlS8cmQR0ECP61Cbu1Jd2Ppyinz8+g==, tarball: file:projects/workspace.tgz} + resolution: {integrity: sha512-Mey3vBH1eqNXrxrhsXI3d66AoIW4hqE34OWI7KN7hqY0nqVzUcex0GHBc78dw7vS+UDkW1Cc2J6rCiLvYQkzfw==, tarball: file:projects/workspace.tgz} id: file:projects/workspace.tgz name: '@rush-temp/workspace' version: 0.0.0 diff --git a/deploy/setup-es-attachment-pipeline.sh b/deploy/setup-es-attachment-pipeline.sh new file mode 100755 index 0000000000..efbc6dd37c --- /dev/null +++ b/deploy/setup-es-attachment-pipeline.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright © 2020, 2021 Anticrm Platform Contributors. +# Copyright © 2021 Hardcore Engineering Inc. +# +# Licensed under the Eclipse Public License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. You may +# obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# + +curl -XPUT "localhost:9200/_ingest/pipeline/attachment?pretty" -H 'Content-Type: application/json' -d' +{ + "description" : "Field for processing file attachments", + "processors" : [ + { + "attachment" : { + "field" : "attachment" + } + } + ] +} +' diff --git a/images/Dockerfile.elastic b/images/Dockerfile.elastic new file mode 100644 index 0000000000..b6fff3e89d --- /dev/null +++ b/images/Dockerfile.elastic @@ -0,0 +1,2 @@ +FROM bitnami/elasticsearch:7.14.0-debian-10-r22 +RUN /opt/bitnami/elasticsearch/bin/elasticsearch-plugin install --batch ingest-attachment diff --git a/images/build-elastic.sh b/images/build-elastic.sh new file mode 100755 index 0000000000..937992be86 --- /dev/null +++ b/images/build-elastic.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Copyright © 2020, 2021 Anticrm Platform Contributors. +# Copyright © 2021 Hardcore Engineering Inc. +# +# Licensed under the Eclipse Public License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. You may +# obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# + +docker build -t anticrm/elasticsearch:7.14.0-debian-10-r22 -f Dockerfile.elastic . diff --git a/packages/core/src/tx.ts b/packages/core/src/tx.ts index 0390436e8a..dd1eb4f765 100644 --- a/packages/core/src/tx.ts +++ b/packages/core/src/tx.ts @@ -174,9 +174,10 @@ export class TxOperations implements Storage { async createDoc ( _class: Ref>, space: Ref, - attributes: Data + attributes: Data, + id?: Ref ): Promise> { - const tx = this.txFactory.createTxCreateDoc(_class, space, attributes) + const tx = this.txFactory.createTxCreateDoc(_class, space, attributes, id) await this.storage.tx(tx) return tx.objectId } diff --git a/plugins/recruit-resources/src/components/CreateCandidate.svelte b/plugins/recruit-resources/src/components/CreateCandidate.svelte index 9fc722ae76..03f233e020 100644 --- a/plugins/recruit-resources/src/components/CreateCandidate.svelte +++ b/plugins/recruit-resources/src/components/CreateCandidate.svelte @@ -49,4 +49,4 @@ } - + diff --git a/plugins/recruit-resources/src/components/DialogHeader.svelte b/plugins/recruit-resources/src/components/DialogHeader.svelte index fa9696afdf..3a1e4cc3c1 100644 --- a/plugins/recruit-resources/src/components/DialogHeader.svelte +++ b/plugins/recruit-resources/src/components/DialogHeader.svelte @@ -16,8 +16,10 @@ diff --git a/plugins/recruit-resources/src/utils.ts b/plugins/recruit-resources/src/utils.ts new file mode 100644 index 0000000000..fec3356b5e --- /dev/null +++ b/plugins/recruit-resources/src/utils.ts @@ -0,0 +1,40 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// Copyright © 2021 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import type { Ref, Doc } from '@anticrm/core' +import { getMetadata } from '@anticrm/platform' + +import login from '@anticrm/login' + +export async function uploadFile(id: Ref, file: File): Promise { + console.log(file) + const uploadUrl = getMetadata(login.metadata.UploadUrl) + + const data = new FormData() + data.append('file', file) + + const url = `${uploadUrl}?id=${id}&space=space` + const resp = await fetch(url, { + method: 'POST', + headers: { + 'Authorization': 'Bearer ' + getMetadata(login.metadata.LoginToken) + }, + body: data + }) + const uuid = await resp.text() + console.log(uuid) + return uuid +} diff --git a/server/core/src/fulltext.ts b/server/core/src/fulltext.ts index b0509b80eb..95f06b420d 100644 --- a/server/core/src/fulltext.ts +++ b/server/core/src/fulltext.ts @@ -16,7 +16,7 @@ import type { TxCreateDoc, Doc, Ref, Class, Obj, Hierarchy, AnyAttribute } from '@anticrm/core' import { TxProcessor, IndexKind } from '@anticrm/core' -import type { IndexedDoc, FullTextAdapter } from './types' +import type { IndexedContent, FullTextAdapter } from './types' // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const NO_INDEX = {} as AnyAttribute @@ -52,7 +52,7 @@ export class FullTextIndex extends TxProcessor { if (attribute === undefined) return const doc = TxProcessor.createDoc2Doc(tx) const content = (doc as any)[attribute.name] - const indexedDoc: IndexedDoc = { + const indexedDoc: IndexedContent = { id: doc._id, _class: doc._class, modifiedBy: doc.modifiedBy, diff --git a/server/core/src/types.ts b/server/core/src/types.ts index 46310d86ef..dbb64525f9 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -39,9 +39,22 @@ export interface IndexedDoc { space: Ref modifiedOn: Timestamp modifiedBy: Ref +} + +/** + * @public + */ +export interface IndexedContent extends IndexedDoc { content: string } +/** + * @public + */ +export interface IndexedAttachment extends IndexedDoc { + attachment: string +} + /** * @public */ diff --git a/server/elastic/src/__tests__/adapter.test.ts b/server/elastic/src/__tests__/adapter.test.ts index 805c408ac7..dacde2397e 100644 --- a/server/elastic/src/__tests__/adapter.test.ts +++ b/server/elastic/src/__tests__/adapter.test.ts @@ -16,18 +16,20 @@ import type { Ref, Doc, Class, Obj, Account, Space } from '@anticrm/core' import { createElasticAdapter } from '../adapter' +import type { IndexedContent } from '@anticrm/server-core' describe('client', () => { it('should create document', async () => { const adapter = await createElasticAdapter('http://localhost:9200/', 'ws1') - await adapter.index({ + const doc: IndexedContent = { id: 'doc1' as Ref, _class: 'class1' as Ref>, modifiedBy: 'andrey' as Ref, modifiedOn: 0, space: 'space1' as Ref, content: 'hey there!' - }) + } + await adapter.index(doc) const hits = await adapter.search({}) console.log(hits) }) diff --git a/server/upload/build.sh b/server/upload/build.sh new file mode 100755 index 0000000000..0301e46926 --- /dev/null +++ b/server/upload/build.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright © 2020, 2021 Anticrm Platform Contributors. +# Copyright © 2021 Hardcore Engineering Inc. +# +# Licensed under the Eclipse Public License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. You may +# obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# + +rushx bundle +rushx docker:build +rushx docker:push diff --git a/server/upload/package.json b/server/upload/package.json index 163ad1ba86..6d1df16ba7 100644 --- a/server/upload/package.json +++ b/server/upload/package.json @@ -21,7 +21,6 @@ "eslint-plugin-promise":"4", "eslint-plugin-node":"11", "eslint":"^7.32.0", - "@types/ws":"^7.4.7", "@types/express":"^4.17.13", "ts-node":"^10.2.0", "@types/express-fileupload":"^1.1.7", @@ -34,7 +33,6 @@ "@anticrm/platform": "~0.6.5", "express": "^4.17.1", "express-fileupload":"^1.2.1", - "aws-sdk":"^2.969.0", "uuid":"^8.3.2", "cors":"^2.8.5", "@anticrm/elastic":"~0.6.0", diff --git a/server/upload/src/app.ts b/server/upload/src/app.ts index bf0e2231bc..a65c46a160 100644 --- a/server/upload/src/app.ts +++ b/server/upload/src/app.ts @@ -20,11 +20,12 @@ import cors from 'cors' import { v4 as uuid } from 'uuid' import { decode } from 'jwt-simple' -import type { Space, Ref, Account, Doc } from '@anticrm/core' -import { TxFactory } from '@anticrm/core' -import type { Token } from '@anticrm/server-core' +import type { Space, Ref, Doc, Account } from '@anticrm/core' +// import { TxFactory } from '@anticrm/core' +import type { Token, IndexedAttachment } from '@anticrm/server-core' +import { createElasticAdapter } from '@anticrm/elastic' import chunter from '@anticrm/chunter' -import { createContributingClient } from '@anticrm/contrib' +// import { createContributingClient } from '@anticrm/contrib' import { Client } from 'minio' @@ -55,19 +56,19 @@ async function minioUpload (minio: Client, workspace: string, file: UploadedFile return id } -async function createAttachment (endpoint: string, token: string, account: Ref, space: Ref, attachmentTo: Ref, collection: string, name: string, file: string): Promise { - const txFactory = new TxFactory(account) - const tx = txFactory.createTxCreateDoc(chunter.class.Attachment, space, { - attachmentTo, - collection, - name, - file - }) - const url = new URL(`/${token}`, endpoint) - const client = await createContributingClient(url.href) - await client.tx(tx) - client.close() -} +// async function createAttachment (endpoint: string, token: string, account: Ref, space: Ref, attachmentTo: Ref, collection: string, name: string, file: string): Promise { +// const txFactory = new TxFactory(account) +// const tx = txFactory.createTxCreateDoc(chunter.class.Attachment, space, { +// attachmentTo, +// collection, +// name, +// file +// }) +// const url = new URL(`/${token}`, endpoint) +// const client = await createContributingClient(url.href) +// await client.tx(tx) +// client.close() +// } /** * @public @@ -81,7 +82,7 @@ export function start (transactorEndpoint: string, elasticUrl: string, minio: Cl // eslint-disable-next-line @typescript-eslint/no-misused-promises app.post('/', async (req, res) => { - const file = req.files?.file + const file = req.files?.file as UploadedFile if (file === undefined) { res.status(400).send() @@ -98,26 +99,37 @@ export function start (transactorEndpoint: string, elasticUrl: string, minio: Cl const token = authHeader.split(' ')[1] const payload = decode(token ?? '', 'secret', false) as Token // const fileId = await awsUpload(file as UploadedFile) - const fileId = await minioUpload(minio, payload.workspace, file as UploadedFile) + const uuid = await minioUpload(minio, payload.workspace, file) + const id = req.query.id as Ref const space = req.query.space as Ref - const attachmentTo = req.query.attachmentTo as Ref - const name = req.query.name as string - const collection = req.query.collection as string - console.log('name', name) + // const name = req.query.name as string - await createAttachment( - transactorEndpoint, - token, - 'core:account:System' as Ref, + // await createAttachment( + // transactorEndpoint, + // token, + // 'core:account:System' as Ref, + // space, + // attachmentTo, + // collection, + // name, + // fileId + // ) + + const elastic = await createElasticAdapter(elasticUrl, payload.workspace) + + const indexedDoc: IndexedAttachment = { + id, + _class: chunter.class.Attachment, space, - attachmentTo, - collection, - name, - fileId - ) + modifiedOn: Date.now(), + modifiedBy: 'core:account:System' as Ref, + attachment: file.data.toString('base64') + } - res.status(200).send() + await elastic.index(indexedDoc) + + res.status(200).send(uuid) } catch (error) { console.log(error) res.status(500).send()