From e16054112ac557855fed94786db8ab21759adb9c Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Mon, 5 Jun 2023 20:11:21 +0600 Subject: [PATCH] UBER-356 Facility to apply initial workspace (#3344) * UBER-356 Facility to apply initial workspace Signed-off-by: Denis Bykhov * Don't allow join to source workspace Signed-off-by: Denis Bykhov --------- Signed-off-by: Denis Bykhov --- dev/docker-compose.yaml | 1 + dev/tool/src/index.ts | 5 ++ pods/account/src/index.ts | 5 ++ server/account/package.json | 1 + server/account/src/index.ts | 11 ++++ server/backup/src/index.ts | 127 ++++++++++++++++++++++++++++++++++++ server/tool/src/plugin.ts | 3 +- 7 files changed, 152 insertions(+), 1 deletion(-) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index c34e7bce51..faf17bc7af 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -60,6 +60,7 @@ services: - MINIO_SECRET_KEY=minioadmin - FRONT_URL=http://front:8080 - SES_URL=http://localhost:8091 + - INIT_WORKSPACE=demo-tracker - MODEL_ENABLED=* restart: unless-stopped collaborator: diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index b0880daec0..dcc0d1b9ab 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -91,6 +91,11 @@ export function devTool ( return elasticUrl } + const initWS = process.env.INIT_WORKSPACE + if (initWS !== undefined) { + setMetadata(toolPlugin.metadata.InitWorkspace, initWS) + } + setMetadata(toolPlugin.metadata.Endpoint, transactorUrl) setMetadata(toolPlugin.metadata.Transactor, transactorUrl) setMetadata(serverToken.metadata.Secret, serverSecret) diff --git a/pods/account/src/index.ts b/pods/account/src/index.ts index 9ca97d3ed6..aef53ee203 100644 --- a/pods/account/src/index.ts +++ b/pods/account/src/index.ts @@ -57,6 +57,11 @@ export function serveAccount (methods: Record, productId setMetadata(account.metadata.FrontURL, frontURL) setMetadata(serverToken.metadata.Secret, serverSecret) + + const initWS = process.env.INIT_WORKSPACE + if (initWS !== undefined) { + setMetadata(toolPlugin.metadata.InitWorkspace, initWS) + } setMetadata(toolPlugin.metadata.Endpoint, endpointUri) setMetadata(toolPlugin.metadata.Transactor, transactorUri) diff --git a/server/account/package.json b/server/account/package.json index 23ac4f60cd..af2a2d2d1e 100644 --- a/server/account/package.json +++ b/server/account/package.json @@ -36,6 +36,7 @@ "@hcengineering/client": "^0.6.11", "ws": "^8.10.0", "@hcengineering/model": "^0.6.4", + "@hcengineering/server-backup": "^0.6.0", "@hcengineering/server-tool": "^0.6.0", "@hcengineering/server-token": "^0.6.4", "@hcengineering/model-all": "^0.6.0", diff --git a/server/account/src/index.ts b/server/account/src/index.ts index 858ace36b0..7d37854297 100644 --- a/server/account/src/index.ts +++ b/server/account/src/index.ts @@ -44,6 +44,7 @@ import platform, { Status, StatusCode } from '@hcengineering/platform' +import { cloneWorkspace } from '@hcengineering/server-backup' import { decodeToken, generateToken } from '@hcengineering/server-token' import toolPlugin, { connect, initModel, upgradeModel } from '@hcengineering/server-tool' import { pbkdf2Sync, randomBytes } from 'crypto' @@ -568,6 +569,12 @@ export async function createWorkspace ( }) .then((e) => e.insertedId.toHexString()) await initModel(getTransactor(), getWorkspaceId(workspace, productId), txes, migrationOperation) + const initWS = getMetadata(toolPlugin.metadata.InitWorkspace) + if (initWS !== undefined) { + if ((await getWorkspace(db, productId, initWS)) !== null) { + await cloneWorkspace(getTransactor(), getWorkspaceId(initWS, productId), getWorkspaceId(workspace, productId)) + } + } return result } @@ -720,6 +727,10 @@ export async function setRole (email: string, workspace: string, productId: stri * @public */ export async function assignWorkspace (db: Db, productId: string, email: string, workspace: string): Promise { + const initWS = getMetadata(toolPlugin.metadata.InitWorkspace) + if (initWS !== undefined && initWS === workspace) { + throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {})) + } const { workspaceId, accountId } = await getWorkspaceAndAccount(db, productId, email, workspace) const account = await db.collection(ACCOUNT_COLLECTION).findOne({ _id: accountId }) diff --git a/server/backup/src/index.ts b/server/backup/src/index.ts index 4872574553..c60b5d4005 100644 --- a/server/backup/src/index.ts +++ b/server/backup/src/index.ts @@ -15,6 +15,7 @@ // import core, { + AttachedDoc, BackupClient, BlobData, Client as CoreClient, @@ -23,6 +24,7 @@ import core, { DOMAIN_MODEL, DOMAIN_TRANSIENT, Ref, + TxCollectionCUD, WorkspaceId } from '@hcengineering/core' import { connect } from '@hcengineering/server-tool' @@ -191,6 +193,131 @@ async function writeChanges (storage: BackupStorage, snapshot: string, changes: ) } +/** + * @public + */ +export async function cloneWorkspace ( + transactorUrl: string, + sourceWorkspaceId: WorkspaceId, + targetWorkspaceId: WorkspaceId, + clearTime: boolean = true +): Promise { + const sourceConnection = (await connect(transactorUrl, sourceWorkspaceId, undefined, { + mode: 'backup' + })) as unknown as CoreClient & BackupClient + const targetConnection = (await connect(transactorUrl, targetWorkspaceId, undefined, { + mode: 'backup' + })) as unknown as CoreClient & BackupClient + try { + const domains = sourceConnection + .getHierarchy() + .domains() + .filter((it) => it !== DOMAIN_TRANSIENT && it !== DOMAIN_MODEL) + + for (const c of domains) { + console.log('clone domain...', c) + + const changes: Snapshot = { + added: new Map(), + updated: new Map(), + removed: [] + } + + let idx: number | undefined + + // update digest tar + const needRetrieveChunks: Ref[][] = [] + + let processed = 0 + let st = Date.now() + // Load all digest from collection. + while (true) { + try { + const it = await sourceConnection.loadChunk(c, idx) + idx = it.idx + + const needRetrieve: Ref[] = [] + + for (const [k, v] of Object.entries(it.docs)) { + processed++ + if (processed % 10000 === 0) { + console.log('processed', processed, Date.now() - st) + st = Date.now() + } + + changes.added.set(k as Ref, v) + needRetrieve.push(k as Ref) + } + if (needRetrieve.length > 0) { + needRetrieveChunks.push(needRetrieve) + } + if (it.finished) { + await sourceConnection.closeChunk(idx) + break + } + } catch (err: any) { + console.error(err) + if (idx !== undefined) { + await sourceConnection.closeChunk(idx) + } + // Try again + idx = undefined + processed = 0 + } + } + while (needRetrieveChunks.length > 0) { + const needRetrieve = needRetrieveChunks.shift() as Ref[] + + console.log('Retrieve chunk:', needRetrieve.length) + let docs: Doc[] = [] + try { + docs = await sourceConnection.loadDocs(c, needRetrieve) + if (clearTime) { + docs = docs.map((p) => { + if (sourceConnection.getHierarchy().isDerived(p._class, core.class.TxCollectionCUD)) { + return { + ...p, + createdBy: core.account.System, + modifiedBy: core.account.System, + modifiedOn: Date.now(), + createdOn: Date.now(), + tx: { + ...(p as TxCollectionCUD).tx, + createdBy: core.account.System, + modifiedBy: core.account.System, + modifiedOn: Date.now(), + createdOn: Date.now() + } + } + } else { + return { + ...p, + createdBy: core.account.System, + modifiedBy: core.account.System, + modifiedOn: Date.now(), + createdOn: Date.now() + } + } + }) + } + await targetConnection.upload(c, docs) + } catch (err: any) { + console.log(err) + // Put back. + needRetrieveChunks.push(needRetrieve) + continue + } + } + } + } catch (err: any) { + console.error(err) + } finally { + console.log('end clone') + await sourceConnection.close() + await targetConnection.close() + } +} + /** * @public */ diff --git a/server/tool/src/plugin.ts b/server/tool/src/plugin.ts index 0040b61b98..14ee3fc055 100644 --- a/server/tool/src/plugin.ts +++ b/server/tool/src/plugin.ts @@ -11,7 +11,8 @@ export const toolId = 'tool' as Plugin const toolPlugin = plugin(toolId, { metadata: { Endpoint: '' as Metadata, - Transactor: '' as Metadata + Transactor: '' as Metadata, + InitWorkspace: '' as Metadata } })