From 94005da7c6f264ddc1dd11acc3c97abe84ce83a4 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 25 Sep 2024 19:05:11 +0700 Subject: [PATCH 1/3] feat: show lost files tool (#6721) --- dev/tool/src/index.ts | 45 ++++++++++++++++++++++++++++++++++++++++- dev/tool/src/storage.ts | 28 +++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 6f98eac8e7..314ba72f37 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -116,7 +116,7 @@ import { fixJsonMarkup, migrateMarkup } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { importNotion } from './notion' import { fixAccountEmails, renameAccount } from './renameAccount' -import { moveFiles, syncFiles } from './storage' +import { moveFiles, showLostFiles, syncFiles } from './storage' const colorConstants = { colorRed: '\u001b[31m', @@ -1230,6 +1230,49 @@ export function devTool ( }) }) + program + .command('show-lost-files') + .option('-w, --workspace ', 'Selected workspace only', '') + .option('--disabled', 'Include disabled workspaces', false) + .option('--all', 'Show all files', false) + .action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => { + const { mongodbUri } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + await withStorage(mongodbUri, async (adapter) => { + try { + let index = 1 + const workspaces = await listWorkspacesPure(db) + workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + + for (const workspace of workspaces) { + if (workspace.disabled === true && !cmd.disabled) { + console.log('ignore disabled workspace', workspace.workspace) + continue + } + + if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { + continue + } + + try { + console.log('start', workspace.workspace, index, '/', workspaces.length) + const workspaceId = getWorkspaceId(workspace.workspace) + const wsDb = getWorkspaceDB(client, { name: workspace.workspace }) + await showLostFiles(toolCtx, workspaceId, wsDb, adapter, { showAll: cmd.all }) + console.log('done', workspace.workspace) + } catch (err) { + console.error(err) + } + + index += 1 + } + } catch (err: any) { + console.error(err) + } + }) + }) + }) + program.command('fix-bw-workspace ').action(async (workspace: string) => { const { mongodbUri } = prepareTools() await withStorage(mongodbUri, async (adapter) => { diff --git a/dev/tool/src/storage.ts b/dev/tool/src/storage.ts index e5a305a910..c70a2d14e0 100644 --- a/dev/tool/src/storage.ts +++ b/dev/tool/src/storage.ts @@ -13,8 +13,11 @@ // limitations under the License. // +import { type Attachment } from '@hcengineering/attachment' import { type Blob, type MeasureContext, type WorkspaceId, RateLimiter } from '@hcengineering/core' +import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment' import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core' +import { type Db } from 'mongodb' import { PassThrough } from 'stream' export interface MoveFilesParams { @@ -93,6 +96,31 @@ export async function moveFiles ( } } +export async function showLostFiles ( + ctx: MeasureContext, + workspaceId: WorkspaceId, + db: Db, + storageAdapter: StorageAdapter, + { showAll }: { showAll: boolean } +): Promise { + const iterator = db.collection(DOMAIN_ATTACHMENT).find({}) + + while (true) { + const attachment = await iterator.next() + if (attachment === null) break + + const { _id, _class, file, name, modifiedOn } = attachment + const date = new Date(modifiedOn).toISOString() + + const stat = await storageAdapter.stat(ctx, workspaceId, file) + if (stat === undefined) { + console.warn('-', date, _class, _id, file, name) + } else if (showAll) { + console.log('+', date, _class, _id, file, name) + } + } +} + async function processAdapter ( ctx: MeasureContext, exAdapter: StorageAdapterEx, From 23c31af23f988db231504c3c0eaa31884e492fde Mon Sep 17 00:00:00 2001 From: Alexander Platov Date: Wed, 25 Sep 2024 20:08:19 +0700 Subject: [PATCH 2/3] Fixed the checkbox twitching in the Inbox (#6723) Signed-off-by: Alexander Platov --- .../src/components/DocNotifyContextCard.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/notification-resources/src/components/DocNotifyContextCard.svelte b/plugins/notification-resources/src/components/DocNotifyContextCard.svelte index 206e845674..926b1eaa61 100644 --- a/plugins/notification-resources/src/components/DocNotifyContextCard.svelte +++ b/plugins/notification-resources/src/components/DocNotifyContextCard.svelte @@ -230,7 +230,7 @@
-
+
{#if archivingPromise !== undefined} {:else} From ea0819e086881ad202a15c8f9329d51a2a1be4e5 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 25 Sep 2024 20:21:23 +0700 Subject: [PATCH 3/3] feat: restore lost markup tool (#6724) Signed-off-by: Alexander Onnikov --- dev/tool/src/index.ts | 34 ++++++++++- dev/tool/src/markup.ts | 133 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 164 insertions(+), 3 deletions(-) diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 314ba72f37..714fc15d6d 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -112,7 +112,7 @@ import { } from './clean' import { changeConfiguration } from './configuration' import { moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db' -import { fixJsonMarkup, migrateMarkup } from './markup' +import { fixJsonMarkup, migrateMarkup, restoreLostMarkup } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { importNotion } from './notion' import { fixAccountEmails, renameAccount } from './renameAccount' @@ -1273,6 +1273,38 @@ export function devTool ( }) }) + program.command('show-lost-markup ').action(async (workspace: string, cmd: any) => { + const { mongodbUri } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + await withStorage(mongodbUri, async (adapter) => { + try { + const workspaceId = getWorkspaceId(workspace) + const token = generateToken(systemAccountEmail, workspaceId) + const endpoint = await getTransactorEndpoint(token) + await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'show' }) + } catch (err: any) { + console.error(err) + } + }) + }) + }) + + program.command('restore-lost-markup ').action(async (workspace: string, cmd: any) => { + const { mongodbUri } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + await withStorage(mongodbUri, async (adapter) => { + try { + const workspaceId = getWorkspaceId(workspace) + const token = generateToken(systemAccountEmail, workspaceId) + const endpoint = await getTransactorEndpoint(token) + await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'restore' }) + } catch (err: any) { + console.error(err) + } + }) + }) + }) + program.command('fix-bw-workspace ').action(async (workspace: string) => { const { mongodbUri } = prepareTools() await withStorage(mongodbUri, async (adapter) => { diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index 837a0cb7f9..13143d5906 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -3,20 +3,28 @@ import core, { type AnyAttribute, type Class, type Client as CoreClient, + type CollaborativeDoc, type Doc, + type DocIndexState, type Domain, type Hierarchy, + type Markup, type MeasureContext, type Ref, + type TxMixin, + type TxCreateDoc, + type TxUpdateDoc, type WorkspaceId, RateLimiter, collaborativeDocParse, - makeCollaborativeDoc + makeCollaborativeDoc, + SortingOrder, + TxProcessor } from '@hcengineering/core' import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo' import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core' import { connect } from '@hcengineering/server-tool' -import { jsonToText, markupToYDoc } from '@hcengineering/text' +import { isEmptyMarkup, jsonToText, markupToYDoc } from '@hcengineering/text' import { type Db, type FindCursor, type MongoClient } from 'mongodb' export async function fixJsonMarkup ( @@ -212,3 +220,124 @@ async function processMigrateMarkupFor ( console.log('processed', processed) } + +export async function restoreLostMarkup ( + ctx: MeasureContext, + workspaceId: WorkspaceId, + transactorUrl: string, + storageAdapter: StorageAdapter, + { command }: { command: 'show' | 'restore' } +): Promise { + const connection = (await connect(transactorUrl, workspaceId, undefined, { + mode: 'backup' + })) as unknown as CoreClient + + try { + const hierarchy = connection.getHierarchy() + const classes = hierarchy.getDescendants(core.class.Doc) + + for (const _class of classes) { + const isAttachedDoc = hierarchy.isDerived(_class, core.class.AttachedDoc) + + const attributes = hierarchy.getAllAttributes(_class) + const attrs = Array.from(attributes.values()).filter((p) => p.type._class === core.class.TypeCollaborativeDoc) + + // ignore classes with no collaborative attributes + if (attrs.length === 0) continue + + const docs = await connection.findAll(_class, { _class }) + for (const doc of docs) { + for (const attr of attrs) { + const value = hierarchy.isMixin(attr.attributeOf) + ? ((doc as any)[attr.attributeOf]?.[attr.name] as CollaborativeDoc) + : ((doc as any)[attr.name] as CollaborativeDoc) + + if (value == null || value === '') continue + + const { documentId } = collaborativeDocParse(value) + const stat = await storageAdapter.stat(ctx, workspaceId, documentId) + if (stat !== undefined) continue + + const query = isAttachedDoc + ? { + 'tx.objectId': doc._id, + 'tx._class': { $in: [core.class.TxCreateDoc, core.class.TxUpdateDoc] } + } + : { + objectId: doc._id + } + + let restored = false + + // try to restore by txes + // we need last tx that modified the attribute + + const txes = await connection.findAll(isAttachedDoc ? core.class.TxCollectionCUD : core.class.TxCUD, query, { + sort: { modifiedOn: SortingOrder.Descending } + }) + for (const tx of txes) { + const innerTx = TxProcessor.extractTx(tx) + + let markup: string | undefined + if (innerTx._class === core.class.TxMixin) { + const mixinTx = innerTx as TxMixin + markup = (mixinTx.attributes as any)[attr.name] + } else if (innerTx._class === core.class.TxCreateDoc) { + const createTx = innerTx as TxCreateDoc + markup = (createTx.attributes as any)[attr.name] + } else if (innerTx._class === core.class.TxUpdateDoc) { + const updateTex = innerTx as TxUpdateDoc + markup = (updateTex.operations as any)[attr.name] + } else { + continue + } + + if (markup === undefined || !markup.startsWith('{')) continue + if (isEmptyMarkup(markup)) continue + + console.log(doc._class, doc._id, attr.name, markup) + if (command === 'restore') { + const ydoc = markupToYDoc(markup, attr.name) + await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx) + } + restored = true + break + } + + if (restored) continue + + // try to restore by doc index state + const docIndexState = await connection.findOne(core.class.DocIndexState, { + _id: doc._id as Ref + }) + if (docIndexState !== undefined) { + // document:class:Document%content#content#base64 + const attrName = `${doc._class}%${attr.name}#content#base64` + const base64: string | undefined = docIndexState.attributes[attrName] + if (base64 !== undefined) { + const text = Buffer.from(base64, 'base64').toString() + if (text !== '') { + const markup: Markup = JSON.stringify({ + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text, marks: [] }] + } + ] + }) + console.log(doc._class, doc._id, attr.name, markup) + if (command === 'restore') { + const ydoc = markupToYDoc(markup, attr.name) + await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx) + } + } + } + } + } + } + } + } finally { + await connection.close() + } +}