diff --git a/dev/tool/package.json b/dev/tool/package.json index 47c913a296..120d8723d6 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -67,6 +67,7 @@ "@hcengineering/client-resources": "^0.6.27", "@hcengineering/contact": "^0.6.24", "@hcengineering/core": "^0.6.32", + "@hcengineering/controlled-documents": "^0.1.0", "@hcengineering/document": "^0.6.0", "@hcengineering/elastic": "^0.6.0", "@hcengineering/lead": "^0.6.0", @@ -75,6 +76,7 @@ "@hcengineering/model-all": "^0.6.0", "@hcengineering/model-attachment": "^0.6.0", "@hcengineering/model-contact": "^0.6.1", + "@hcengineering/model-controlled-documents": "^0.1.0", "@hcengineering/model-document": "^0.6.0", "@hcengineering/model-recruit": "^0.6.0", "@hcengineering/model-telegram": "^0.6.0", diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index cb53d67e7b..882a2a14a2 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -122,7 +122,7 @@ import { copyToDatalake, moveFiles, showLostFiles } from './storage' import { getModelVersion } from '@hcengineering/model-all' import { type DatalakeConfig, DatalakeService, createDatalakeClient } from '@hcengineering/datalake' import { S3Service, type S3Config } from '@hcengineering/s3' -import { restoreWikiContentMongo } from './markup' +import { restoreControlledDocContentMongo, restoreWikiContentMongo } from './markup' const colorConstants = { colorRed: '\u001b[31m', @@ -1263,6 +1263,8 @@ export function devTool ( .sort((a, b) => b.lastVisit - a.lastVisit) }) + console.log('found workspaces', workspaces.length) + await withStorage(async (storageAdapter) => { await withDatabase(dbUrl, async (db) => { const mongodbUri = getMongoDBUrl() @@ -1293,6 +1295,62 @@ export function devTool ( }) }) + program + .command('restore-controlled-content-mongo') + .description('restore controlled document contents') + .option('-w, --workspace ', 'Selected workspace only', '') + .option('-d, --dryrun', 'Dry run', false) + .option('-f, --force', 'Force update', false) + .action(async (cmd: { workspace: string, dryrun: boolean, force: boolean }) => { + const params = { + dryRun: cmd.dryrun + } + + const { dbUrl, version } = prepareTools() + + let workspaces: Workspace[] = [] + const accountUrl = getAccountDBUrl() + await withDatabase(accountUrl, async (db) => { + workspaces = await listWorkspacesPure(db) + workspaces = workspaces + .filter((p) => p.mode !== 'archived') + .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) + .sort((a, b) => b.lastVisit - a.lastVisit) + }) + + console.log('found workspaces', workspaces.length) + + await withStorage(async (storageAdapter) => { + await withDatabase(dbUrl, async (db) => { + const mongodbUri = getMongoDBUrl() + const client = getMongoClient(mongodbUri) + const _client = await client.getClient() + + try { + const count = workspaces.length + let index = 0 + for (const workspace of workspaces) { + index++ + + toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) + + if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) { + console.log(`upgrade to ${versionToString(version)} is required`) + continue + } + + const workspaceId = getWorkspaceId(workspace.workspace) + const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) + + await restoreControlledDocContentMongo(toolCtx, wsDb, workspaceId, storageAdapter, params) + } + } finally { + client.close() + } + }) + }) + }) + program .command('confirm-email ') .description('confirm user email') diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index 4712f2ad31..6aa9e1b1a6 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -14,9 +14,21 @@ // import { loadCollabYdoc, saveCollabYdoc, yDocCopyXmlField } from '@hcengineering/collaboration' -import { type MeasureContext, type WorkspaceId, makeCollabYdocId } from '@hcengineering/core' +import core, { + type Blob, + type Doc, + type MeasureContext, + type Ref, + type TxCreateDoc, + type WorkspaceId, + DOMAIN_TX, + makeCollabYdocId, + makeDocCollabId +} from '@hcengineering/core' import document, { type Document } from '@hcengineering/document' +import documents from '@hcengineering/controlled-documents' import { DOMAIN_DOCUMENT } from '@hcengineering/model-document' +import { DOMAIN_DOCUMENTS } from '@hcengineering/model-controlled-documents' import { type StorageAdapter } from '@hcengineering/server-core' import { type Db } from 'mongodb' @@ -43,7 +55,7 @@ export async function restoreWikiContentMongo ( try { while (true) { const doc = await iterator.next() - if (doc === null) return + if (doc === null) break processedCnt++ if (processedCnt % 100 === 0) { @@ -88,3 +100,111 @@ export async function restoreWikiContentMongo ( await iterator.close() } } + +export interface RestoreControlledDocContentParams { + dryRun: boolean +} + +export async function restoreControlledDocContentMongo ( + ctx: MeasureContext, + db: Db, + workspaceId: WorkspaceId, + storageAdapter: StorageAdapter, + params: RestoreWikiContentParams +): Promise { + const iterator = db.collection(DOMAIN_DOCUMENTS).find({ + _class: { + $in: [documents.class.ControlledDocument, documents.class.ControlledDocumentSnapshot] + } + }) + + let processedCnt = 0 + let restoredCnt = 0 + + function printStats (): void { + console.log('...processed', processedCnt, 'restored', restoredCnt) + } + + try { + while (true) { + const doc = await iterator.next() + if (doc === null) break + + const restored = await restoreControlledDocContentForDoc( + ctx, + db, + workspaceId, + storageAdapter, + params, + doc, + 'content' + ) + if (restored) { + restoredCnt++ + } + + processedCnt++ + if (processedCnt % 100 === 0) { + printStats() + } + } + } finally { + printStats() + await iterator.close() + } +} + +export async function restoreControlledDocContentForDoc ( + ctx: MeasureContext, + db: Db, + workspaceId: WorkspaceId, + storageAdapter: StorageAdapter, + params: RestoreWikiContentParams, + doc: Doc, + attribute: string +): Promise { + const tx = await db.collection>(DOMAIN_TX).findOne({ + _class: core.class.TxCreateDoc, + objectId: doc._id, + objectClass: doc._class + }) + + // It is expected that tx contains attribute with content in old collaborative doc format + // the original value here looks like '65b7f82f4d422b89d4cbdd6f:HEAD:0' + const attribures = tx?.attributes ?? {} + const value = (attribures as any)[attribute] as string + if (value == null || !value.includes(':')) { + console.log('no content to restore', doc._class, doc._id) + return false + } + + const currentYdocId = value.split(':')[0] as Ref + const ydocId = makeCollabYdocId(makeDocCollabId(doc, attribute)) + + // Ensure that we don't have new content in storage + const stat = await storageAdapter.stat(ctx, workspaceId, ydocId) + if (stat !== undefined) { + console.log('content already restored', doc._class, doc._id, ydocId) + return false + } + + console.log('restoring content', doc._id, currentYdocId, '-->', ydocId) + if (!params.dryRun) { + try { + const stat = await storageAdapter.stat(ctx, workspaceId, currentYdocId) + if (stat === undefined) { + console.log('no content to restore', doc._class, doc._id, ydocId) + return false + } + + const data = await storageAdapter.read(ctx, workspaceId, currentYdocId) + const buffer = Buffer.concat(data as any) + await storageAdapter.put(ctx, workspaceId, ydocId, buffer, 'application/ydoc', buffer.length) + } catch (err: any) { + console.error('failed to restore content for', doc._class, doc._id, err) + return false + } + } + + return true +} diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index f922129f28..5ece09f075 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -376,15 +376,9 @@ async function processMigrateJsonForDoc ( await retry(5, async () => { const stat = await storageAdapter.stat(ctx, workspaceId, currentYdocId) if (stat !== undefined) { - const buffer = await storageAdapter.read(ctx, workspaceId, currentYdocId) - await storageAdapter.put( - ctx, - workspaceId, - ydocId, - Buffer.concat(buffer as any), - 'application/ydoc', - buffer.length - ) + const data = await storageAdapter.read(ctx, workspaceId, currentYdocId) + const buffer = Buffer.concat(data as any) + await storageAdapter.put(ctx, workspaceId, ydocId, buffer, 'application/ydoc', buffer.length) } }) }