tool: restore controlled docs content (#7423)

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>

dev/tool/package.json

@@ -67,6 +67,7 @@
     "@hcengineering/client-resources": "^0.6.27",
     "@hcengineering/contact": "^0.6.24",
     "@hcengineering/core": "^0.6.32",
+    "@hcengineering/controlled-documents": "^0.1.0",
     "@hcengineering/document": "^0.6.0",
     "@hcengineering/elastic": "^0.6.0",
     "@hcengineering/lead": "^0.6.0",
@@ -75,6 +76,7 @@
     "@hcengineering/model-all": "^0.6.0",
     "@hcengineering/model-attachment": "^0.6.0",
     "@hcengineering/model-contact": "^0.6.1",
+    "@hcengineering/model-controlled-documents": "^0.1.0",
     "@hcengineering/model-document": "^0.6.0",
     "@hcengineering/model-recruit": "^0.6.0",
     "@hcengineering/model-telegram": "^0.6.0",

dev/tool/src/index.ts

@@ -122,7 +122,7 @@ import { copyToDatalake, moveFiles, showLostFiles } from './storage'
 import { getModelVersion } from '@hcengineering/model-all'
 import { type DatalakeConfig, DatalakeService, createDatalakeClient } from '@hcengineering/datalake'
 import { S3Service, type S3Config } from '@hcengineering/s3'
-import { restoreWikiContentMongo } from './markup'
+import { restoreControlledDocContentMongo, restoreWikiContentMongo } from './markup'
 
 const colorConstants = {
   colorRed: '\u001b[31m',
@@ -1263,6 +1263,8 @@ export function devTool (
           .sort((a, b) => b.lastVisit - a.lastVisit)
       })
 
+      console.log('found workspaces', workspaces.length)
+
       await withStorage(async (storageAdapter) => {
         await withDatabase(dbUrl, async (db) => {
           const mongodbUri = getMongoDBUrl()
@@ -1293,6 +1295,62 @@ export function devTool (
       })
     })
 
+  program
+    .command('restore-controlled-content-mongo')
+    .description('restore controlled document contents')
+    .option('-w, --workspace <workspace>', 'Selected workspace only', '')
+    .option('-d, --dryrun', 'Dry run', false)
+    .option('-f, --force', 'Force update', false)
+    .action(async (cmd: { workspace: string, dryrun: boolean, force: boolean }) => {
+      const params = {
+        dryRun: cmd.dryrun
+      }
+
+      const { dbUrl, version } = prepareTools()
+
+      let workspaces: Workspace[] = []
+      const accountUrl = getAccountDBUrl()
+      await withDatabase(accountUrl, async (db) => {
+        workspaces = await listWorkspacesPure(db)
+        workspaces = workspaces
+          .filter((p) => p.mode !== 'archived')
+          .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace)
+          .sort((a, b) => b.lastVisit - a.lastVisit)
+      })
+
+      console.log('found workspaces', workspaces.length)
+
+      await withStorage(async (storageAdapter) => {
+        await withDatabase(dbUrl, async (db) => {
+          const mongodbUri = getMongoDBUrl()
+          const client = getMongoClient(mongodbUri)
+          const _client = await client.getClient()
+
+          try {
+            const count = workspaces.length
+            let index = 0
+            for (const workspace of workspaces) {
+              index++
+
+              toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count })
+
+              if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) {
+                console.log(`upgrade to ${versionToString(version)} is required`)
+                continue
+              }
+
+              const workspaceId = getWorkspaceId(workspace.workspace)
+              const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace })
+
+              await restoreControlledDocContentMongo(toolCtx, wsDb, workspaceId, storageAdapter, params)
+            }
+          } finally {
+            client.close()
+          }
+        })
+      })
+    })
+
   program
     .command('confirm-email <email>')
     .description('confirm user email')

dev/tool/src/markup.ts

@@ -14,9 +14,21 @@
 //
 
 import { loadCollabYdoc, saveCollabYdoc, yDocCopyXmlField } from '@hcengineering/collaboration'
-import { type MeasureContext, type WorkspaceId, makeCollabYdocId } from '@hcengineering/core'
+import core, {
+  type Blob,
+  type Doc,
+  type MeasureContext,
+  type Ref,
+  type TxCreateDoc,
+  type WorkspaceId,
+  DOMAIN_TX,
+  makeCollabYdocId,
+  makeDocCollabId
+} from '@hcengineering/core'
 import document, { type Document } from '@hcengineering/document'
+import documents from '@hcengineering/controlled-documents'
 import { DOMAIN_DOCUMENT } from '@hcengineering/model-document'
+import { DOMAIN_DOCUMENTS } from '@hcengineering/model-controlled-documents'
 import { type StorageAdapter } from '@hcengineering/server-core'
 import { type Db } from 'mongodb'
 
@@ -43,7 +55,7 @@ export async function restoreWikiContentMongo (
   try {
     while (true) {
       const doc = await iterator.next()
-      if (doc === null) return
+      if (doc === null) break
 
       processedCnt++
       if (processedCnt % 100 === 0) {
@@ -88,3 +100,111 @@ export async function restoreWikiContentMongo (
     await iterator.close()
   }
 }
+
+export interface RestoreControlledDocContentParams {
+  dryRun: boolean
+}
+
+export async function restoreControlledDocContentMongo (
+  ctx: MeasureContext,
+  db: Db,
+  workspaceId: WorkspaceId,
+  storageAdapter: StorageAdapter,
+  params: RestoreWikiContentParams
+): Promise<void> {
+  const iterator = db.collection<Doc>(DOMAIN_DOCUMENTS).find({
+    _class: {
+      $in: [documents.class.ControlledDocument, documents.class.ControlledDocumentSnapshot]
+    }
+  })
+
+  let processedCnt = 0
+  let restoredCnt = 0
+
+  function printStats (): void {
+    console.log('...processed', processedCnt, 'restored', restoredCnt)
+  }
+
+  try {
+    while (true) {
+      const doc = await iterator.next()
+      if (doc === null) break
+
+      const restored = await restoreControlledDocContentForDoc(
+        ctx,
+        db,
+        workspaceId,
+        storageAdapter,
+        params,
+        doc,
+        'content'
+      )
+      if (restored) {
+        restoredCnt++
+      }
+
+      processedCnt++
+      if (processedCnt % 100 === 0) {
+        printStats()
+      }
+    }
+  } finally {
+    printStats()
+    await iterator.close()
+  }
+}
+
+export async function restoreControlledDocContentForDoc (
+  ctx: MeasureContext,
+  db: Db,
+  workspaceId: WorkspaceId,
+  storageAdapter: StorageAdapter,
+  params: RestoreWikiContentParams,
+  doc: Doc,
+  attribute: string
+): Promise<boolean> {
+  const tx = await db.collection<TxCreateDoc<Doc>>(DOMAIN_TX).findOne({
+    _class: core.class.TxCreateDoc,
+    objectId: doc._id,
+    objectClass: doc._class
+  })
+
+  // It is expected that tx contains attribute with content in old collaborative doc format
+  // the original value here looks like '65b7f82f4d422b89d4cbdd6f:HEAD:0'
+  const attribures = tx?.attributes ?? {}
+  const value = (attribures as any)[attribute] as string
+  if (value == null || !value.includes(':')) {
+    console.log('no content to restore', doc._class, doc._id)
+    return false
+  }
+
+  const currentYdocId = value.split(':')[0] as Ref<Blob>
+  const ydocId = makeCollabYdocId(makeDocCollabId(doc, attribute))
+
+  // Ensure that we don't have new content in storage
+  const stat = await storageAdapter.stat(ctx, workspaceId, ydocId)
+  if (stat !== undefined) {
+    console.log('content already restored', doc._class, doc._id, ydocId)
+    return false
+  }
+
+  console.log('restoring content', doc._id, currentYdocId, '-->', ydocId)
+  if (!params.dryRun) {
+    try {
+      const stat = await storageAdapter.stat(ctx, workspaceId, currentYdocId)
+      if (stat === undefined) {
+        console.log('no content to restore', doc._class, doc._id, ydocId)
+        return false
+      }
+
+      const data = await storageAdapter.read(ctx, workspaceId, currentYdocId)
+      const buffer = Buffer.concat(data as any)
+      await storageAdapter.put(ctx, workspaceId, ydocId, buffer, 'application/ydoc', buffer.length)
+    } catch (err: any) {
+      console.error('failed to restore content for', doc._class, doc._id, err)
+      return false
+    }
+  }
+
+  return true
+}

models/core/src/migration.ts

@@ -376,15 +376,9 @@ async function processMigrateJsonForDoc (
         await retry(5, async () => {
           const stat = await storageAdapter.stat(ctx, workspaceId, currentYdocId)
           if (stat !== undefined) {
-            const buffer = await storageAdapter.read(ctx, workspaceId, currentYdocId)
-            await storageAdapter.put(
-              ctx,
-              workspaceId,
-              ydocId,
-              Buffer.concat(buffer as any),
-              'application/ydoc',
-              buffer.length
-            )
+            const data = await storageAdapter.read(ctx, workspaceId, currentYdocId)
+            const buffer = Buffer.concat(data as any)
+            await storageAdapter.put(ctx, workspaceId, ydocId, buffer, 'application/ydoc', buffer.length)
           }
         })
       }