Fix markdown migration tool (#6407)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/tool/package.json

@@ -16,6 +16,7 @@
     "_phase:docker-staging": "rushx docker:staging",
     "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --minify --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) > bundle/bundle.js",
     "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/tool",
+    "docker:tbuild": "docker build -t hardcoreeng/tool . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool",
     "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging",
     "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool",
     "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --max-old-space-size=18000 ./bundle/bundle.js",

dev/tool/src/index.ts

@@ -1300,19 +1300,19 @@ export function devTool (
       await withDatabase(mongodbUri, async (db, client) => {
         await withStorage(mongodbUri, async (adapter) => {
           const workspaces = await listWorkspacesPure(db)
+          let index = 0
           for (const workspace of workspaces) {
             if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) {
               continue
             }
 
             const wsId = getWorkspaceId(workspace.workspace)
-            const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsId), 'external')
+            console.log('processing workspace', workspace.workspace, index, workspaces.length)
 
-            console.log('processing workspace', workspace.workspace)
+            await migrateMarkup(toolCtx, adapter, wsId, client, mongodbUri, parseInt(cmd.concurrency))
-
-            await migrateMarkup(toolCtx, adapter, wsId, client, endpoint, parseInt(cmd.concurrency))
 
             console.log('...done', workspace.workspace)
+            index++
           }
         })
       })

dev/tool/src/markup.ts

@@ -15,7 +15,7 @@ import core, {
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
 import { type StorageAdapter } from '@hcengineering/server-core'
-import { connect } from '@hcengineering/server-tool'
+import { connect, fetchModelFromMongo } from '@hcengineering/server-tool'
 import { jsonToText, markupToYDoc } from '@hcengineering/text'
 import { type Db, type FindCursor, type MongoClient } from 'mongodb'
 
@@ -120,47 +120,39 @@ export async function migrateMarkup (
   storageAdapter: StorageAdapter,
   workspaceId: WorkspaceId,
   client: MongoClient,
-  transactorUrl: string,
+  mongodbUri: string,
   concurrency: number
 ): Promise<void> {
-  const connection = (await connect(transactorUrl, workspaceId, undefined, {
+  const { hierarchy } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId)
-    mode: 'backup'
-  })) as unknown as CoreClient
-
-  const hierarchy = connection.getHierarchy()
 
   const workspaceDb = client.db(workspaceId.name)
 
-  try {
+  const classes = hierarchy.getDescendants(core.class.Doc)
-    const classes = hierarchy.getDescendants(core.class.Doc)
+  for (const _class of classes) {
-    for (const _class of classes) {
+    const domain = hierarchy.findDomain(_class)
-      const domain = hierarchy.findDomain(_class)
+    if (domain === undefined) continue
-      if (domain === undefined) continue
 
-      const allAttributes = hierarchy.getAllAttributes(_class)
+    const allAttributes = hierarchy.getAllAttributes(_class)
-      const attributes = Array.from(allAttributes.values()).filter((attribute) => {
+    const attributes = Array.from(allAttributes.values()).filter((attribute) => {
-        return hierarchy.isDerived(attribute.type._class, 'core:class:TypeCollaborativeMarkup' as Ref<Class<Doc>>)
+      return hierarchy.isDerived(attribute.type._class, 'core:class:TypeCollaborativeMarkup' as Ref<Class<Doc>>)
-      })
+    })
 
-      if (attributes.length === 0) continue
+    if (attributes.length === 0) continue
-      if (hierarchy.isMixin(_class) && attributes.every((p) => p.attributeOf !== _class)) continue
+    if (hierarchy.isMixin(_class) && attributes.every((p) => p.attributeOf !== _class)) continue
 
-      const collection = workspaceDb.collection(domain)
+    const collection = workspaceDb.collection(domain)
 
-      const filter = hierarchy.isMixin(_class) ? { [_class]: { $exists: true } } : { _class }
+    const filter = hierarchy.isMixin(_class) ? { [_class]: { $exists: true } } : { _class }
 
-      const count = await collection.countDocuments(filter)
+    const count = await collection.countDocuments(filter)
-      const iterator = collection.find<Doc>(filter)
+    const iterator = collection.find<Doc>(filter)
 
-      try {
+    try {
-        console.log('processing', _class, '->', count)
+      console.log('processing', _class, '->', count)
-        await processMigrateMarkupFor(ctx, hierarchy, storageAdapter, workspaceId, attributes, iterator, concurrency)
+      await processMigrateMarkupFor(ctx, hierarchy, storageAdapter, workspaceId, attributes, iterator, concurrency)
-      } finally {
+    } finally {
-        await iterator.close()
+      await iterator.close()
-      }
     }
-  } finally {
-    await connection.close()
   }
 }
 

server/tool/src/index.ts

@@ -469,7 +469,7 @@ async function prepareMigrationClient (
   return { migrateClient, migrateState }
 }
 
-async function fetchModelFromMongo (
+export async function fetchModelFromMongo (
   ctx: MeasureContext,
   mongodbUri: string,
   workspaceId: WorkspaceId,