Add json markup tool (#5511)

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>

dev/tool/package.json

@@ -132,6 +132,7 @@
     "@hcengineering/setting": "^0.6.11",
     "@hcengineering/tags": "^0.6.12",
     "@hcengineering/task": "^0.6.13",
+    "@hcengineering/text": "^0.6.1",
     "@hcengineering/telegram": "^0.6.14",
     "@hcengineering/tracker": "^0.6.13",
     "commander": "^8.1.0",

dev/tool/src/index.ts

@@ -82,6 +82,7 @@ import { changeConfiguration } from './configuration'
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
 import { openAIConfig } from './openai'
 import { fixAccountEmails, renameAccount } from './renameAccount'
+import { fixJsonMarkup } from './markup'
 
 const colorConstants = {
   colorRed: '\u001b[31m',
@@ -881,6 +882,14 @@ export function devTool (
       }
     )
 
+  program
+    .command('fix-json-markup <workspace>')
+    .description('fixes double converted json markup')
+    .action(async (workspace: string) => {
+      const { mongodbUri, storageAdapter } = prepareTools()
+      await fixJsonMarkup(toolCtx, mongodbUri, storageAdapter, getWorkspaceId(workspace, productId), transactorUrl)
+    })
+
   extendProgram?.(program)
 
   program.parse(process.argv)

dev/tool/src/markup.ts

@@ -0,0 +1,114 @@
+import core, {
+  type AnyAttribute,
+  type Class,
+  type Client as CoreClient,
+  type Doc,
+  type Domain,
+  type MeasureContext,
+  type Ref,
+  type WorkspaceId,
+  getCollaborativeDocId
+} from '@hcengineering/core'
+import { getWorkspaceDB } from '@hcengineering/mongo'
+import { type StorageAdapter } from '@hcengineering/server-core'
+import { connect } from '@hcengineering/server-tool'
+import { jsonToText } from '@hcengineering/text'
+import { type Db, MongoClient } from 'mongodb'
+
+export async function fixJsonMarkup (
+  ctx: MeasureContext,
+  mongoUrl: string,
+  storageAdapter: StorageAdapter,
+  workspaceId: WorkspaceId,
+  transactorUrl: string
+): Promise<void> {
+  const connection = (await connect(transactorUrl, workspaceId, undefined, {
+    mode: 'backup'
+  })) as unknown as CoreClient
+  const hierarchy = connection.getHierarchy()
+
+  const client = new MongoClient(mongoUrl)
+  const db = getWorkspaceDB(client, workspaceId)
+
+  try {
+    const classes = hierarchy.getDescendants(core.class.Doc)
+    for (const _class of classes) {
+      const domain = hierarchy.findDomain(_class)
+      if (domain === undefined) continue
+
+      const attributes = hierarchy.getAllAttributes(_class)
+      const filtered = Array.from(attributes.values()).filter((attribute) => {
+        return (
+          hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) ||
+          hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)
+        )
+      })
+      if (filtered.length === 0) continue
+
+      await processFixJsonMarkupFor(ctx, domain, _class, filtered, workspaceId, db, storageAdapter)
+    }
+  } finally {
+    await client.close()
+    await connection.close()
+  }
+}
+
+async function processFixJsonMarkupFor (
+  ctx: MeasureContext,
+  domain: Domain,
+  _class: Ref<Class<Doc>>,
+  attributes: AnyAttribute[],
+  workspaceId: WorkspaceId,
+  db: Db,
+  storageAdapter: StorageAdapter
+): Promise<void> {
+  console.log('processing', domain, _class)
+
+  const collection = db.collection<Doc>(domain)
+  const docs = await collection.find({ _class }).toArray()
+  for (const doc of docs) {
+    const update: Record<string, any> = {}
+    const remove = []
+
+    for (const attribute of attributes) {
+      try {
+        const value = (doc as any)[attribute.name]
+        if (value != null) {
+          let res = value
+          while (true) {
+            try {
+              const json = JSON.parse(res)
+              const text = jsonToText(json)
+              JSON.parse(text)
+              res = text
+            } catch {
+              break
+            }
+          }
+          if (res !== value) {
+            update[attribute.name] = res
+            remove.push(getCollaborativeDocId(doc._id, attribute.name))
+          }
+        }
+      } catch {}
+    }
+
+    if (Object.keys(update).length > 0) {
+      try {
+        await collection.updateOne({ _id: doc._id }, { $set: update })
+      } catch (err) {
+        console.error('failed to update document', doc._class, doc._id, err)
+      }
+    }
+
+    if (remove.length > 0) {
+      try {
+        await storageAdapter.remove(ctx, workspaceId, remove)
+      } catch (err) {
+        console.error('failed to remove objects from storage', doc._class, doc._id, remove, err)
+      }
+    }
+  }
+
+  console.log('...processed', docs.length)
+}

models/text-editor/src/migration.ts

@@ -23,7 +23,7 @@ import {
   type MigrationIterator,
   type MigrationUpgradeClient
 } from '@hcengineering/model'
-import { htmlToMarkup, jsonToPmNode } from '@hcengineering/text'
+import { htmlToMarkup, jsonToPmNode, jsonToText } from '@hcengineering/text'
 
 async function migrateMarkup (client: MigrationClient): Promise<void> {
   const hierarchy = client.hierarchy
@@ -147,9 +147,10 @@ async function processFixMigrateMarkupFor (
             let res = value
             while ((res as string).includes('\\"type\\"')) {
               try {
-                const textOrJson = JSON.parse(res)
+                const json = JSON.parse(res)
-                JSON.parse(textOrJson)
+                const text = jsonToText(json)
-                res = textOrJson
+                JSON.parse(text)
+                res = text
               } catch {
                 break
               }