Merge remote-tracking branch 'origin/develop' into staging

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>

dev/tool/src/index.ts

@@ -112,11 +112,11 @@ import {
 } from './clean'
 import { changeConfiguration } from './configuration'
 import { moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db'
-import { fixJsonMarkup, migrateMarkup } from './markup'
+import { fixJsonMarkup, migrateMarkup, restoreLostMarkup } from './markup'
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
 import { importNotion } from './notion'
 import { fixAccountEmails, renameAccount } from './renameAccount'
-import { moveFiles, syncFiles } from './storage'
+import { moveFiles, showLostFiles, syncFiles } from './storage'
 
 const colorConstants = {
   colorRed: '\u001b[31m',
@@ -1230,6 +1230,81 @@ export function devTool (
       })
     })
 
+  program
+    .command('show-lost-files')
+    .option('-w, --workspace <workspace>', 'Selected workspace only', '')
+    .option('--disabled', 'Include disabled workspaces', false)
+    .option('--all', 'Show all files', false)
+    .action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => {
+      const { mongodbUri } = prepareTools()
+      await withDatabase(mongodbUri, async (db, client) => {
+        await withStorage(mongodbUri, async (adapter) => {
+          try {
+            let index = 1
+            const workspaces = await listWorkspacesPure(db)
+            workspaces.sort((a, b) => b.lastVisit - a.lastVisit)
+
+            for (const workspace of workspaces) {
+              if (workspace.disabled === true && !cmd.disabled) {
+                console.log('ignore disabled workspace', workspace.workspace)
+                continue
+              }
+
+              if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) {
+                continue
+              }
+
+              try {
+                console.log('start', workspace.workspace, index, '/', workspaces.length)
+                const workspaceId = getWorkspaceId(workspace.workspace)
+                const wsDb = getWorkspaceDB(client, { name: workspace.workspace })
+                await showLostFiles(toolCtx, workspaceId, wsDb, adapter, { showAll: cmd.all })
+                console.log('done', workspace.workspace)
+              } catch (err) {
+                console.error(err)
+              }
+
+              index += 1
+            }
+          } catch (err: any) {
+            console.error(err)
+          }
+        })
+      })
+    })
+
+  program.command('show-lost-markup <workspace>').action(async (workspace: string, cmd: any) => {
+    const { mongodbUri } = prepareTools()
+    await withDatabase(mongodbUri, async (db, client) => {
+      await withStorage(mongodbUri, async (adapter) => {
+        try {
+          const workspaceId = getWorkspaceId(workspace)
+          const token = generateToken(systemAccountEmail, workspaceId)
+          const endpoint = await getTransactorEndpoint(token)
+          await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'show' })
+        } catch (err: any) {
+          console.error(err)
+        }
+      })
+    })
+  })
+
+  program.command('restore-lost-markup <workspace>').action(async (workspace: string, cmd: any) => {
+    const { mongodbUri } = prepareTools()
+    await withDatabase(mongodbUri, async (db, client) => {
+      await withStorage(mongodbUri, async (adapter) => {
+        try {
+          const workspaceId = getWorkspaceId(workspace)
+          const token = generateToken(systemAccountEmail, workspaceId)
+          const endpoint = await getTransactorEndpoint(token)
+          await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'restore' })
+        } catch (err: any) {
+          console.error(err)
+        }
+      })
+    })
+  })
+
   program.command('fix-bw-workspace <workspace>').action(async (workspace: string) => {
     const { mongodbUri } = prepareTools()
     await withStorage(mongodbUri, async (adapter) => {

dev/tool/src/markup.ts

@@ -3,20 +3,28 @@ import core, {
   type AnyAttribute,
   type Class,
   type Client as CoreClient,
+  type CollaborativeDoc,
   type Doc,
+  type DocIndexState,
   type Domain,
   type Hierarchy,
+  type Markup,
   type MeasureContext,
   type Ref,
+  type TxMixin,
+  type TxCreateDoc,
+  type TxUpdateDoc,
   type WorkspaceId,
   RateLimiter,
   collaborativeDocParse,
-  makeCollaborativeDoc
+  makeCollaborativeDoc,
+  SortingOrder,
+  TxProcessor
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
 import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core'
 import { connect } from '@hcengineering/server-tool'
-import { jsonToText, markupToYDoc } from '@hcengineering/text'
+import { isEmptyMarkup, jsonToText, markupToYDoc } from '@hcengineering/text'
 import { type Db, type FindCursor, type MongoClient } from 'mongodb'
 
 export async function fixJsonMarkup (
@@ -212,3 +220,124 @@ async function processMigrateMarkupFor (
 
   console.log('processed', processed)
 }
+
+export async function restoreLostMarkup (
+  ctx: MeasureContext,
+  workspaceId: WorkspaceId,
+  transactorUrl: string,
+  storageAdapter: StorageAdapter,
+  { command }: { command: 'show' | 'restore' }
+): Promise<void> {
+  const connection = (await connect(transactorUrl, workspaceId, undefined, {
+    mode: 'backup'
+  })) as unknown as CoreClient
+
+  try {
+    const hierarchy = connection.getHierarchy()
+    const classes = hierarchy.getDescendants(core.class.Doc)
+
+    for (const _class of classes) {
+      const isAttachedDoc = hierarchy.isDerived(_class, core.class.AttachedDoc)
+
+      const attributes = hierarchy.getAllAttributes(_class)
+      const attrs = Array.from(attributes.values()).filter((p) => p.type._class === core.class.TypeCollaborativeDoc)
+
+      // ignore classes with no collaborative attributes
+      if (attrs.length === 0) continue
+
+      const docs = await connection.findAll(_class, { _class })
+      for (const doc of docs) {
+        for (const attr of attrs) {
+          const value = hierarchy.isMixin(attr.attributeOf)
+            ? ((doc as any)[attr.attributeOf]?.[attr.name] as CollaborativeDoc)
+            : ((doc as any)[attr.name] as CollaborativeDoc)
+
+          if (value == null || value === '') continue
+
+          const { documentId } = collaborativeDocParse(value)
+          const stat = await storageAdapter.stat(ctx, workspaceId, documentId)
+          if (stat !== undefined) continue
+
+          const query = isAttachedDoc
+            ? {
+                'tx.objectId': doc._id,
+                'tx._class': { $in: [core.class.TxCreateDoc, core.class.TxUpdateDoc] }
+              }
+            : {
+                objectId: doc._id
+              }
+
+          let restored = false
+
+          // try to restore by txes
+          // we need last tx that modified the attribute
+
+          const txes = await connection.findAll(isAttachedDoc ? core.class.TxCollectionCUD : core.class.TxCUD, query, {
+            sort: { modifiedOn: SortingOrder.Descending }
+          })
+          for (const tx of txes) {
+            const innerTx = TxProcessor.extractTx(tx)
+
+            let markup: string | undefined
+            if (innerTx._class === core.class.TxMixin) {
+              const mixinTx = innerTx as TxMixin<Doc, Doc>
+              markup = (mixinTx.attributes as any)[attr.name]
+            } else if (innerTx._class === core.class.TxCreateDoc) {
+              const createTx = innerTx as TxCreateDoc<Doc>
+              markup = (createTx.attributes as any)[attr.name]
+            } else if (innerTx._class === core.class.TxUpdateDoc) {
+              const updateTex = innerTx as TxUpdateDoc<Doc>
+              markup = (updateTex.operations as any)[attr.name]
+            } else {
+              continue
+            }
+
+            if (markup === undefined || !markup.startsWith('{')) continue
+            if (isEmptyMarkup(markup)) continue
+
+            console.log(doc._class, doc._id, attr.name, markup)
+            if (command === 'restore') {
+              const ydoc = markupToYDoc(markup, attr.name)
+              await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx)
+            }
+            restored = true
+            break
+          }
+
+          if (restored) continue
+
+          // try to restore by doc index state
+          const docIndexState = await connection.findOne(core.class.DocIndexState, {
+            _id: doc._id as Ref<DocIndexState>
+          })
+          if (docIndexState !== undefined) {
+            // document:class:Document%content#content#base64
+            const attrName = `${doc._class}%${attr.name}#content#base64`
+            const base64: string | undefined = docIndexState.attributes[attrName]
+            if (base64 !== undefined) {
+              const text = Buffer.from(base64, 'base64').toString()
+              if (text !== '') {
+                const markup: Markup = JSON.stringify({
+                  type: 'doc',
+                  content: [
+                    {
+                      type: 'paragraph',
+                      content: [{ type: 'text', text, marks: [] }]
+                    }
+                  ]
+                })
+                console.log(doc._class, doc._id, attr.name, markup)
+                if (command === 'restore') {
+                  const ydoc = markupToYDoc(markup, attr.name)
+                  await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx)
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  } finally {
+    await connection.close()
+  }
+}

dev/tool/src/storage.ts

@@ -13,8 +13,11 @@
 // limitations under the License.
 //
 
+import { type Attachment } from '@hcengineering/attachment'
 import { type Blob, type MeasureContext, type WorkspaceId, RateLimiter } from '@hcengineering/core'
+import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment'
 import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core'
+import { type Db } from 'mongodb'
 import { PassThrough } from 'stream'
 
 export interface MoveFilesParams {
@@ -93,6 +96,31 @@ export async function moveFiles (
   }
 }
 
+export async function showLostFiles (
+  ctx: MeasureContext,
+  workspaceId: WorkspaceId,
+  db: Db,
+  storageAdapter: StorageAdapter,
+  { showAll }: { showAll: boolean }
+): Promise<void> {
+  const iterator = db.collection<Attachment>(DOMAIN_ATTACHMENT).find({})
+
+  while (true) {
+    const attachment = await iterator.next()
+    if (attachment === null) break
+
+    const { _id, _class, file, name, modifiedOn } = attachment
+    const date = new Date(modifiedOn).toISOString()
+
+    const stat = await storageAdapter.stat(ctx, workspaceId, file)
+    if (stat === undefined) {
+      console.warn('-', date, _class, _id, file, name)
+    } else if (showAll) {
+      console.log('+', date, _class, _id, file, name)
+    }
+  }
+}
+
 async function processAdapter (
   ctx: MeasureContext,
   exAdapter: StorageAdapterEx,

plugins/notification-resources/src/components/DocNotifyContextCard.svelte

@@ -230,7 +230,7 @@
       </div>
 
       <div class="actions clear-mins">
-        <div class="flex-center">
+        <div class="flex-center min-w-6">
           {#if archivingPromise !== undefined}
             <Spinner size="small" />
           {:else}