EQMS-1376 Fix content tool (#7589)

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>

dev/tool/src/index.ts

@@ -69,7 +69,7 @@ import {
   registerTxAdapterFactory
 } from '@hcengineering/server-pipeline'
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
-import { FileModelLogger } from '@hcengineering/server-tool'
+import { FileModelLogger, buildModel } from '@hcengineering/server-tool'
 import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
 import path from 'path'
 
@@ -143,7 +143,7 @@ import {
   moveFromMongoToPG,
   moveWorkspaceFromMongoToPG
 } from './db'
-import { restoreControlledDocContentMongo, restoreWikiContentMongo } from './markup'
+import { restoreControlledDocContentMongo, restoreWikiContentMongo, restoreMarkupRefsMongo } from './markup'
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
 import { fixAccountEmails, renameAccount } from './renameAccount'
 import { copyToDatalake, moveFiles, showLostFiles } from './storage'
@@ -1345,6 +1345,61 @@ export function devTool (
       })
     })
 
+  program
+    .command('restore-markup-ref-mongo')
+    .description('restore markup document content refs')
+    .option('-w, --workspace <workspace>', 'Selected workspace only', '')
+    .option('-f, --force', 'Force update', false)
+    .action(async (cmd: { workspace: string, force: boolean }) => {
+      const { txes, version } = prepareTools()
+
+      const { hierarchy } = await buildModel(toolCtx, txes)
+
+      let workspaces: Workspace[] = []
+      await withAccountDatabase(async (db) => {
+        workspaces = await listWorkspacesPure(db)
+        workspaces = workspaces
+          .filter((p) => isActiveMode(p.mode))
+          .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace)
+          .sort((a, b) => b.lastVisit - a.lastVisit)
+      })
+
+      console.log('found workspaces', workspaces.length)
+
+      await withStorage(async (storageAdapter) => {
+        const mongodbUri = getMongoDBUrl()
+        const client = getMongoClient(mongodbUri)
+        const _client = await client.getClient()
+
+        try {
+          const count = workspaces.length
+          let index = 0
+          for (const workspace of workspaces) {
+            index++
+
+            toolCtx.info('processing workspace', {
+              workspace: workspace.workspace,
+              version: workspace.version,
+              index,
+              count
+            })
+
+            if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) {
+              console.log(`upgrade to ${versionToString(version)} is required`)
+              continue
+            }
+
+            const workspaceId = getWorkspaceId(workspace.workspace)
+            const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace })
+
+            await restoreMarkupRefsMongo(toolCtx, wsDb, workspaceId, hierarchy, storageAdapter)
+          }
+        } finally {
+          client.close()
+        }
+      })
+    })
+
   program
     .command('confirm-email <email>')
     .description('confirm user email')

dev/tool/src/markup.ts

@@ -13,10 +13,17 @@
 // limitations under the License.
 //
 
-import { loadCollabYdoc, saveCollabYdoc, yDocCopyXmlField } from '@hcengineering/collaboration'
+import {
+  loadCollabYdoc,
+  saveCollabJson,
+  saveCollabYdoc,
+  yDocCopyXmlField,
+  yDocFromBuffer
+} from '@hcengineering/collaboration'
 import core, {
   type Blob,
   type Doc,
+  type Hierarchy,
   type MeasureContext,
   type Ref,
   type TxCreateDoc,
@@ -24,6 +31,7 @@ import core, {
   type WorkspaceId,
   DOMAIN_TX,
   SortingOrder,
+  makeCollabId,
   makeCollabYdocId,
   makeDocCollabId
 } from '@hcengineering/core'
@@ -290,3 +298,65 @@ export async function restoreControlledDocContentForDoc (
 
   return true
 }
+
+export async function restoreMarkupRefsMongo (
+  ctx: MeasureContext,
+  db: Db,
+  workspaceId: WorkspaceId,
+  hierarchy: Hierarchy,
+  storageAdapter: StorageAdapter
+): Promise<void> {
+  const classes = hierarchy.getDescendants(core.class.Doc)
+  for (const _class of classes) {
+    const domain = hierarchy.findDomain(_class)
+    if (domain === undefined) continue
+
+    const allAttributes = hierarchy.getAllAttributes(_class)
+    const attributes = Array.from(allAttributes.values()).filter((attribute) => {
+      return hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc)
+    })
+
+    if (attributes.length === 0) continue
+    if (hierarchy.isMixin(_class) && attributes.every((p) => p.attributeOf !== _class)) continue
+
+    ctx.info('processing', { _class, attributes: attributes.map((p) => p.name) })
+
+    const collection = db.collection<Doc>(domain)
+    const iterator = collection.find({ _class })
+    try {
+      while (true) {
+        const doc = await iterator.next()
+        if (doc === null) {
+          break
+        }
+
+        for (const attribute of attributes) {
+          const isMixin = hierarchy.isMixin(attribute.attributeOf)
+
+          const attributeName = isMixin ? `${attribute.attributeOf}.${attribute.name}` : attribute.name
+
+          const value = isMixin
+            ? ((doc as any)[attribute.attributeOf]?.[attribute.name] as string)
+            : ((doc as any)[attribute.name] as string)
+
+          if (typeof value === 'string') {
+            continue
+          }
+
+          const collabId = makeCollabId(doc._class, doc._id, attribute.name)
+          const ydocId = makeCollabYdocId(collabId)
+
+          try {
+            const buffer = await storageAdapter.read(ctx, workspaceId, ydocId)
+            const ydoc = yDocFromBuffer(Buffer.concat(buffer as any))
+
+            const jsonId = await saveCollabJson(ctx, storageAdapter, workspaceId, collabId, ydoc)
+            await collection.updateOne({ _id: doc._id }, { $set: { [attributeName]: jsonId } })
+          } catch {}
+        }
+      }
+    } finally {
+      await iterator.close()
+    }
+  }
+}

plugins/text-editor-resources/src/provider/hocuspocus.ts

@@ -33,7 +33,7 @@ export class HocuspocusCollabProvider extends HocuspocusProvider implements Prov
     const parameters: Record<string, any> = {}
 
     const content = configuration.parameters?.content
-    if (content !== null && content !== '') {
+    if (content !== null && content !== undefined && content !== '') {
       parameters.content = content
     }