From c2d0aad760e6810ddb375779dc9f369fc9a19fbe Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Wed, 28 Aug 2024 00:12:42 +0700 Subject: [PATCH] Fix markdown migration tool (#6407) Signed-off-by: Andrey Sobolev --- dev/tool/package.json | 1 + dev/tool/src/index.ts | 8 +++---- dev/tool/src/markup.ts | 52 +++++++++++++++++----------------------- server/tool/src/index.ts | 2 +- 4 files changed, 28 insertions(+), 35 deletions(-) diff --git a/dev/tool/package.json b/dev/tool/package.json index 6d0e67c600..79dd4519fa 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -16,6 +16,7 @@ "_phase:docker-staging": "rushx docker:staging", "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --minify --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) > bundle/bundle.js", "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/tool", + "docker:tbuild": "docker build -t hardcoreeng/tool . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool", "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --max-old-space-size=18000 ./bundle/bundle.js", diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 56671c74f9..3a13e78bb2 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -1300,19 +1300,19 @@ export function devTool ( await withDatabase(mongodbUri, async (db, client) => { await withStorage(mongodbUri, async (adapter) => { const workspaces = await listWorkspacesPure(db) + let index = 0 for (const workspace of workspaces) { if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { continue } const wsId = getWorkspaceId(workspace.workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsId), 'external') + console.log('processing workspace', workspace.workspace, index, workspaces.length) - console.log('processing workspace', workspace.workspace) - - await migrateMarkup(toolCtx, adapter, wsId, client, endpoint, parseInt(cmd.concurrency)) + await migrateMarkup(toolCtx, adapter, wsId, client, mongodbUri, parseInt(cmd.concurrency)) console.log('...done', workspace.workspace) + index++ } }) }) diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index 8b889614d9..67afdd44db 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -15,7 +15,7 @@ import core, { } from '@hcengineering/core' import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo' import { type StorageAdapter } from '@hcengineering/server-core' -import { connect } from '@hcengineering/server-tool' +import { connect, fetchModelFromMongo } from '@hcengineering/server-tool' import { jsonToText, markupToYDoc } from '@hcengineering/text' import { type Db, type FindCursor, type MongoClient } from 'mongodb' @@ -120,47 +120,39 @@ export async function migrateMarkup ( storageAdapter: StorageAdapter, workspaceId: WorkspaceId, client: MongoClient, - transactorUrl: string, + mongodbUri: string, concurrency: number ): Promise { - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup' - })) as unknown as CoreClient - - const hierarchy = connection.getHierarchy() + const { hierarchy } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId) const workspaceDb = client.db(workspaceId.name) - try { - const classes = hierarchy.getDescendants(core.class.Doc) - for (const _class of classes) { - const domain = hierarchy.findDomain(_class) - if (domain === undefined) continue + const classes = hierarchy.getDescendants(core.class.Doc) + for (const _class of classes) { + const domain = hierarchy.findDomain(_class) + if (domain === undefined) continue - const allAttributes = hierarchy.getAllAttributes(_class) - const attributes = Array.from(allAttributes.values()).filter((attribute) => { - return hierarchy.isDerived(attribute.type._class, 'core:class:TypeCollaborativeMarkup' as Ref>) - }) + const allAttributes = hierarchy.getAllAttributes(_class) + const attributes = Array.from(allAttributes.values()).filter((attribute) => { + return hierarchy.isDerived(attribute.type._class, 'core:class:TypeCollaborativeMarkup' as Ref>) + }) - if (attributes.length === 0) continue - if (hierarchy.isMixin(_class) && attributes.every((p) => p.attributeOf !== _class)) continue + if (attributes.length === 0) continue + if (hierarchy.isMixin(_class) && attributes.every((p) => p.attributeOf !== _class)) continue - const collection = workspaceDb.collection(domain) + const collection = workspaceDb.collection(domain) - const filter = hierarchy.isMixin(_class) ? { [_class]: { $exists: true } } : { _class } + const filter = hierarchy.isMixin(_class) ? { [_class]: { $exists: true } } : { _class } - const count = await collection.countDocuments(filter) - const iterator = collection.find(filter) + const count = await collection.countDocuments(filter) + const iterator = collection.find(filter) - try { - console.log('processing', _class, '->', count) - await processMigrateMarkupFor(ctx, hierarchy, storageAdapter, workspaceId, attributes, iterator, concurrency) - } finally { - await iterator.close() - } + try { + console.log('processing', _class, '->', count) + await processMigrateMarkupFor(ctx, hierarchy, storageAdapter, workspaceId, attributes, iterator, concurrency) + } finally { + await iterator.close() } - } finally { - await connection.close() } } diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index f29c8f38e5..4a358558cc 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -469,7 +469,7 @@ async function prepareMigrationClient ( return { migrateClient, migrateState } } -async function fetchModelFromMongo ( +export async function fetchModelFromMongo ( ctx: MeasureContext, mongodbUri: string, workspaceId: WorkspaceId,