diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 6f98eac8e7..714fc15d6d 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -112,11 +112,11 @@ import { } from './clean' import { changeConfiguration } from './configuration' import { moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db' -import { fixJsonMarkup, migrateMarkup } from './markup' +import { fixJsonMarkup, migrateMarkup, restoreLostMarkup } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { importNotion } from './notion' import { fixAccountEmails, renameAccount } from './renameAccount' -import { moveFiles, syncFiles } from './storage' +import { moveFiles, showLostFiles, syncFiles } from './storage' const colorConstants = { colorRed: '\u001b[31m', @@ -1230,6 +1230,81 @@ export function devTool ( }) }) + program + .command('show-lost-files') + .option('-w, --workspace ', 'Selected workspace only', '') + .option('--disabled', 'Include disabled workspaces', false) + .option('--all', 'Show all files', false) + .action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => { + const { mongodbUri } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + await withStorage(mongodbUri, async (adapter) => { + try { + let index = 1 + const workspaces = await listWorkspacesPure(db) + workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + + for (const workspace of workspaces) { + if (workspace.disabled === true && !cmd.disabled) { + console.log('ignore disabled workspace', workspace.workspace) + continue + } + + if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { + continue + } + + try { + console.log('start', workspace.workspace, index, '/', workspaces.length) + const workspaceId = getWorkspaceId(workspace.workspace) + const wsDb = getWorkspaceDB(client, { name: workspace.workspace }) + await showLostFiles(toolCtx, workspaceId, wsDb, adapter, { showAll: cmd.all }) + console.log('done', workspace.workspace) + } catch (err) { + console.error(err) + } + + index += 1 + } + } catch (err: any) { + console.error(err) + } + }) + }) + }) + + program.command('show-lost-markup ').action(async (workspace: string, cmd: any) => { + const { mongodbUri } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + await withStorage(mongodbUri, async (adapter) => { + try { + const workspaceId = getWorkspaceId(workspace) + const token = generateToken(systemAccountEmail, workspaceId) + const endpoint = await getTransactorEndpoint(token) + await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'show' }) + } catch (err: any) { + console.error(err) + } + }) + }) + }) + + program.command('restore-lost-markup ').action(async (workspace: string, cmd: any) => { + const { mongodbUri } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + await withStorage(mongodbUri, async (adapter) => { + try { + const workspaceId = getWorkspaceId(workspace) + const token = generateToken(systemAccountEmail, workspaceId) + const endpoint = await getTransactorEndpoint(token) + await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'restore' }) + } catch (err: any) { + console.error(err) + } + }) + }) + }) + program.command('fix-bw-workspace ').action(async (workspace: string) => { const { mongodbUri } = prepareTools() await withStorage(mongodbUri, async (adapter) => { diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index 837a0cb7f9..13143d5906 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -3,20 +3,28 @@ import core, { type AnyAttribute, type Class, type Client as CoreClient, + type CollaborativeDoc, type Doc, + type DocIndexState, type Domain, type Hierarchy, + type Markup, type MeasureContext, type Ref, + type TxMixin, + type TxCreateDoc, + type TxUpdateDoc, type WorkspaceId, RateLimiter, collaborativeDocParse, - makeCollaborativeDoc + makeCollaborativeDoc, + SortingOrder, + TxProcessor } from '@hcengineering/core' import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo' import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core' import { connect } from '@hcengineering/server-tool' -import { jsonToText, markupToYDoc } from '@hcengineering/text' +import { isEmptyMarkup, jsonToText, markupToYDoc } from '@hcengineering/text' import { type Db, type FindCursor, type MongoClient } from 'mongodb' export async function fixJsonMarkup ( @@ -212,3 +220,124 @@ async function processMigrateMarkupFor ( console.log('processed', processed) } + +export async function restoreLostMarkup ( + ctx: MeasureContext, + workspaceId: WorkspaceId, + transactorUrl: string, + storageAdapter: StorageAdapter, + { command }: { command: 'show' | 'restore' } +): Promise { + const connection = (await connect(transactorUrl, workspaceId, undefined, { + mode: 'backup' + })) as unknown as CoreClient + + try { + const hierarchy = connection.getHierarchy() + const classes = hierarchy.getDescendants(core.class.Doc) + + for (const _class of classes) { + const isAttachedDoc = hierarchy.isDerived(_class, core.class.AttachedDoc) + + const attributes = hierarchy.getAllAttributes(_class) + const attrs = Array.from(attributes.values()).filter((p) => p.type._class === core.class.TypeCollaborativeDoc) + + // ignore classes with no collaborative attributes + if (attrs.length === 0) continue + + const docs = await connection.findAll(_class, { _class }) + for (const doc of docs) { + for (const attr of attrs) { + const value = hierarchy.isMixin(attr.attributeOf) + ? ((doc as any)[attr.attributeOf]?.[attr.name] as CollaborativeDoc) + : ((doc as any)[attr.name] as CollaborativeDoc) + + if (value == null || value === '') continue + + const { documentId } = collaborativeDocParse(value) + const stat = await storageAdapter.stat(ctx, workspaceId, documentId) + if (stat !== undefined) continue + + const query = isAttachedDoc + ? { + 'tx.objectId': doc._id, + 'tx._class': { $in: [core.class.TxCreateDoc, core.class.TxUpdateDoc] } + } + : { + objectId: doc._id + } + + let restored = false + + // try to restore by txes + // we need last tx that modified the attribute + + const txes = await connection.findAll(isAttachedDoc ? core.class.TxCollectionCUD : core.class.TxCUD, query, { + sort: { modifiedOn: SortingOrder.Descending } + }) + for (const tx of txes) { + const innerTx = TxProcessor.extractTx(tx) + + let markup: string | undefined + if (innerTx._class === core.class.TxMixin) { + const mixinTx = innerTx as TxMixin + markup = (mixinTx.attributes as any)[attr.name] + } else if (innerTx._class === core.class.TxCreateDoc) { + const createTx = innerTx as TxCreateDoc + markup = (createTx.attributes as any)[attr.name] + } else if (innerTx._class === core.class.TxUpdateDoc) { + const updateTex = innerTx as TxUpdateDoc + markup = (updateTex.operations as any)[attr.name] + } else { + continue + } + + if (markup === undefined || !markup.startsWith('{')) continue + if (isEmptyMarkup(markup)) continue + + console.log(doc._class, doc._id, attr.name, markup) + if (command === 'restore') { + const ydoc = markupToYDoc(markup, attr.name) + await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx) + } + restored = true + break + } + + if (restored) continue + + // try to restore by doc index state + const docIndexState = await connection.findOne(core.class.DocIndexState, { + _id: doc._id as Ref + }) + if (docIndexState !== undefined) { + // document:class:Document%content#content#base64 + const attrName = `${doc._class}%${attr.name}#content#base64` + const base64: string | undefined = docIndexState.attributes[attrName] + if (base64 !== undefined) { + const text = Buffer.from(base64, 'base64').toString() + if (text !== '') { + const markup: Markup = JSON.stringify({ + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text, marks: [] }] + } + ] + }) + console.log(doc._class, doc._id, attr.name, markup) + if (command === 'restore') { + const ydoc = markupToYDoc(markup, attr.name) + await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx) + } + } + } + } + } + } + } + } finally { + await connection.close() + } +} diff --git a/dev/tool/src/storage.ts b/dev/tool/src/storage.ts index e5a305a910..c70a2d14e0 100644 --- a/dev/tool/src/storage.ts +++ b/dev/tool/src/storage.ts @@ -13,8 +13,11 @@ // limitations under the License. // +import { type Attachment } from '@hcengineering/attachment' import { type Blob, type MeasureContext, type WorkspaceId, RateLimiter } from '@hcengineering/core' +import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment' import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core' +import { type Db } from 'mongodb' import { PassThrough } from 'stream' export interface MoveFilesParams { @@ -93,6 +96,31 @@ export async function moveFiles ( } } +export async function showLostFiles ( + ctx: MeasureContext, + workspaceId: WorkspaceId, + db: Db, + storageAdapter: StorageAdapter, + { showAll }: { showAll: boolean } +): Promise { + const iterator = db.collection(DOMAIN_ATTACHMENT).find({}) + + while (true) { + const attachment = await iterator.next() + if (attachment === null) break + + const { _id, _class, file, name, modifiedOn } = attachment + const date = new Date(modifiedOn).toISOString() + + const stat = await storageAdapter.stat(ctx, workspaceId, file) + if (stat === undefined) { + console.warn('-', date, _class, _id, file, name) + } else if (showAll) { + console.log('+', date, _class, _id, file, name) + } + } +} + async function processAdapter ( ctx: MeasureContext, exAdapter: StorageAdapterEx, diff --git a/plugins/notification-resources/src/components/DocNotifyContextCard.svelte b/plugins/notification-resources/src/components/DocNotifyContextCard.svelte index 206e845674..926b1eaa61 100644 --- a/plugins/notification-resources/src/components/DocNotifyContextCard.svelte +++ b/plugins/notification-resources/src/components/DocNotifyContextCard.svelte @@ -230,7 +230,7 @@
-
+
{#if archivingPromise !== undefined} {:else}