From bb29cebdd91395e68c48e2f332a49fe38121bd04 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Sat, 4 May 2024 22:14:14 +0700 Subject: [PATCH] Add json markup tool (#5511) Signed-off-by: Alexander Onnikov --- dev/tool/package.json | 1 + dev/tool/src/index.ts | 9 +++ dev/tool/src/markup.ts | 114 ++++++++++++++++++++++++++++ models/text-editor/src/migration.ts | 9 ++- 4 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 dev/tool/src/markup.ts diff --git a/dev/tool/package.json b/dev/tool/package.json index 377101f14b..e016b7e9f4 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -132,6 +132,7 @@ "@hcengineering/setting": "^0.6.11", "@hcengineering/tags": "^0.6.12", "@hcengineering/task": "^0.6.13", + "@hcengineering/text": "^0.6.1", "@hcengineering/telegram": "^0.6.14", "@hcengineering/tracker": "^0.6.13", "commander": "^8.1.0", diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 289a76af60..90ff2e66d2 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -82,6 +82,7 @@ import { changeConfiguration } from './configuration' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { openAIConfig } from './openai' import { fixAccountEmails, renameAccount } from './renameAccount' +import { fixJsonMarkup } from './markup' const colorConstants = { colorRed: '\u001b[31m', @@ -881,6 +882,14 @@ export function devTool ( } ) + program + .command('fix-json-markup ') + .description('fixes double converted json markup') + .action(async (workspace: string) => { + const { mongodbUri, storageAdapter } = prepareTools() + await fixJsonMarkup(toolCtx, mongodbUri, storageAdapter, getWorkspaceId(workspace, productId), transactorUrl) + }) + extendProgram?.(program) program.parse(process.argv) diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts new file mode 100644 index 0000000000..b1f512a341 --- /dev/null +++ b/dev/tool/src/markup.ts @@ -0,0 +1,114 @@ +import core, { + type AnyAttribute, + type Class, + type Client as CoreClient, + type Doc, + type Domain, + type MeasureContext, + type Ref, + type WorkspaceId, + getCollaborativeDocId +} from '@hcengineering/core' +import { getWorkspaceDB } from '@hcengineering/mongo' +import { type StorageAdapter } from '@hcengineering/server-core' +import { connect } from '@hcengineering/server-tool' +import { jsonToText } from '@hcengineering/text' +import { type Db, MongoClient } from 'mongodb' + +export async function fixJsonMarkup ( + ctx: MeasureContext, + mongoUrl: string, + storageAdapter: StorageAdapter, + workspaceId: WorkspaceId, + transactorUrl: string +): Promise { + const connection = (await connect(transactorUrl, workspaceId, undefined, { + mode: 'backup' + })) as unknown as CoreClient + const hierarchy = connection.getHierarchy() + + const client = new MongoClient(mongoUrl) + const db = getWorkspaceDB(client, workspaceId) + + try { + const classes = hierarchy.getDescendants(core.class.Doc) + for (const _class of classes) { + const domain = hierarchy.findDomain(_class) + if (domain === undefined) continue + + const attributes = hierarchy.getAllAttributes(_class) + const filtered = Array.from(attributes.values()).filter((attribute) => { + return ( + hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) || + hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup) + ) + }) + if (filtered.length === 0) continue + + await processFixJsonMarkupFor(ctx, domain, _class, filtered, workspaceId, db, storageAdapter) + } + } finally { + await client.close() + await connection.close() + } +} + +async function processFixJsonMarkupFor ( + ctx: MeasureContext, + domain: Domain, + _class: Ref>, + attributes: AnyAttribute[], + workspaceId: WorkspaceId, + db: Db, + storageAdapter: StorageAdapter +): Promise { + console.log('processing', domain, _class) + + const collection = db.collection(domain) + const docs = await collection.find({ _class }).toArray() + for (const doc of docs) { + const update: Record = {} + const remove = [] + + for (const attribute of attributes) { + try { + const value = (doc as any)[attribute.name] + if (value != null) { + let res = value + while (true) { + try { + const json = JSON.parse(res) + const text = jsonToText(json) + JSON.parse(text) + res = text + } catch { + break + } + } + if (res !== value) { + update[attribute.name] = res + remove.push(getCollaborativeDocId(doc._id, attribute.name)) + } + } + } catch {} + } + + if (Object.keys(update).length > 0) { + try { + await collection.updateOne({ _id: doc._id }, { $set: update }) + } catch (err) { + console.error('failed to update document', doc._class, doc._id, err) + } + } + + if (remove.length > 0) { + try { + await storageAdapter.remove(ctx, workspaceId, remove) + } catch (err) { + console.error('failed to remove objects from storage', doc._class, doc._id, remove, err) + } + } + } + + console.log('...processed', docs.length) +} diff --git a/models/text-editor/src/migration.ts b/models/text-editor/src/migration.ts index 0412bea7cf..ad667d5f99 100644 --- a/models/text-editor/src/migration.ts +++ b/models/text-editor/src/migration.ts @@ -23,7 +23,7 @@ import { type MigrationIterator, type MigrationUpgradeClient } from '@hcengineering/model' -import { htmlToMarkup, jsonToPmNode } from '@hcengineering/text' +import { htmlToMarkup, jsonToPmNode, jsonToText } from '@hcengineering/text' async function migrateMarkup (client: MigrationClient): Promise { const hierarchy = client.hierarchy @@ -147,9 +147,10 @@ async function processFixMigrateMarkupFor ( let res = value while ((res as string).includes('\\"type\\"')) { try { - const textOrJson = JSON.parse(res) - JSON.parse(textOrJson) - res = textOrJson + const json = JSON.parse(res) + const text = jsonToText(json) + JSON.parse(text) + res = text } catch { break }