UBERF-8461: Fix migration do not clean backup info (#6913)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/tool/src/clean.ts

@@ -14,13 +14,13 @@
 //
 
 import { getAccountDB, listWorkspacesRaw } from '@hcengineering/account'
-import attachment from '@hcengineering/attachment'
 import calendar from '@hcengineering/calendar'
 import chunter, { type ChatMessage } from '@hcengineering/chunter'
 import { loadCollaborativeDoc, saveCollaborativeDoc, yDocToBuffer } from '@hcengineering/collaboration'
 import contact from '@hcengineering/contact'
 import core, {
   type ArrOf,
+  type AttachedDoc,
   type BackupClient,
   type Class,
   ClassifierKind,
@@ -45,11 +45,14 @@ import core, {
   SortingOrder,
   type Status,
   type StatusCategory,
+  type Tx,
   type TxCUD,
+  type TxCollectionCUD,
   type TxCreateDoc,
   type TxMixin,
   TxOperations,
   TxProcessor,
+  type TxRemoveDoc,
   type TxUpdateDoc,
   type WorkspaceId,
   generateId,
@@ -93,30 +96,6 @@ export async function cleanWorkspace (
 
     const hierarchy = ops.getHierarchy()
 
-    const attachments = await ops.findAll(attachment.class.Attachment, {})
-
-    const contacts = await ops.findAll(contact.class.Contact, {})
-
-    const files = new Set(
-      attachments.map((it) => it.file as string).concat(contacts.map((it) => it.avatar).filter((it) => it) as string[])
-    )
-
-    const minioList = await storageAdapter.listStream(ctx, workspaceId)
-    const toClean: string[] = []
-    while (true) {
-      const mvFiles = await minioList.next()
-      if (mvFiles.length === 0) {
-        break
-      }
-
-      for (const mv of mvFiles) {
-        if (!files.has(mv._id)) {
-          toClean.push(mv._id)
-        }
-      }
-    }
-    await storageAdapter.remove(ctx, workspaceId, toClean)
-
     if (opt.recruit) {
       const contacts = await ops.findAll(recruit.mixin.Candidate, {})
       console.log('removing Talents', contacts.length)
@@ -134,13 +113,6 @@ export async function cleanWorkspace (
         const t2 = Date.now()
         console.log('remove time:', t2 - t, filter.length)
       }
-
-      // const vacancies = await ops.findAll(recruit.class.Vacancy, {})
-      // console.log('removing vacancies', vacancies.length)
-      // for (const c of vacancies) {
-      //   console.log('Remove', c.name)
-      //   await ops.remove(c)
-      // }
     }
 
     if (opt.tracker) {
@@ -166,12 +138,39 @@ export async function cleanWorkspace (
       const db = getWorkspaceMongoDB(_client, workspaceId)
 
       if (opt.removedTx) {
-        const txes = await db.collection(DOMAIN_TX).find({}).toArray()
+        let processed = 0
+        const iterator = db.collection(DOMAIN_TX).find({})
+        while (true) {
+          const txes: Tx[] = []
 
-        for (const tx of txes) {
+          const doc = await iterator.next()
-          if (tx._class === core.class.TxRemoveDoc) {
+          if (doc == null) {
-            // We need to remove all update and create operations for document
+            break
-            await db.collection(DOMAIN_TX).deleteMany({ objectId: tx.objectId })
+          }
+          txes.push(doc as unknown as Tx)
+          if (iterator.bufferedCount() > 0) {
+            txes.push(...(iterator.readBufferedDocuments() as unknown as Tx[]))
+          }
+
+          for (const tx of txes) {
+            if (tx._class === core.class.TxRemoveDoc) {
+              // We need to remove all update and create operations for document
+              await db.collection(DOMAIN_TX).deleteMany({ objectId: (tx as TxRemoveDoc<Doc>).objectId })
+              processed++
+            }
+            if (
+              tx._class === core.class.TxCollectionCUD &&
+              (tx as TxCollectionCUD<Doc, AttachedDoc>).tx._class === core.class.TxRemoveDoc
+            ) {
+              // We need to remove all update and create operations for document
+              await db.collection(DOMAIN_TX).deleteMany({
+                'tx.objectId': ((tx as TxCollectionCUD<Doc, AttachedDoc>).tx as TxRemoveDoc<Doc>).objectId
+              })
+              processed++
+            }
+          }
+          if (processed % 1000 === 0) {
+            console.log('processed', processed)
           }
         }
       }

models/activity/src/migration.ts

@@ -221,6 +221,12 @@ export const activityOperation: MigrateOperation = {
       {
         state: 'migrate-activity-markup',
         func: migrateActivityMarkup
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_ACTIVITY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },

models/attachment/src/migration.ts

@@ -13,9 +13,24 @@
 // limitations under the License.
 //
 
-import { type MigrateOperation, type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model'
+import {
+  tryMigrate,
+  type MigrateOperation,
+  type MigrationClient,
+  type MigrationUpgradeClient
+} from '@hcengineering/model'
+import { attachmentId, DOMAIN_ATTACHMENT } from '.'
 
 export const attachmentOperation: MigrateOperation = {
-  async migrate (client: MigrationClient): Promise<void> {},
+  async migrate (client: MigrationClient): Promise<void> {
+    await tryMigrate(client, attachmentId, [
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_ATTACHMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
+      }
+    ])
+  },
   async upgrade (state: Map<string, Set<string>>, client: () => Promise<MigrationUpgradeClient>): Promise<void> {}
 }

models/chunter/src/migration.ts

@@ -362,6 +362,12 @@ export const chunterOperation: MigrateOperation = {
             'attributeUpdates.attrKey': 'members'
           })
         }
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_CHUNTER, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },

models/contact/src/migration.ts

@@ -1,5 +1,6 @@
 //
 
+import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact'
 import {
   type Class,
   type Doc,
@@ -24,9 +25,8 @@ import {
 import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity'
 import core, { DOMAIN_SPACE } from '@hcengineering/model-core'
 import { DOMAIN_VIEW } from '@hcengineering/model-view'
-import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact'
 
-import contact, { contactId, DOMAIN_CONTACT } from './index'
+import contact, { contactId, DOMAIN_CHANNEL, DOMAIN_CONTACT } from './index'
 
 async function createEmployeeEmail (client: TxOperations): Promise<void> {
   const employees = await client.findAll(contact.mixin.Employee, {})
@@ -300,6 +300,13 @@ export const contactOperation: MigrateOperation = {
       {
         state: 'create-person-spaces-v1',
         func: createPersonSpaces
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_CONTACT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+          await client.update(DOMAIN_CHANNEL, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },

models/core/src/migration.ts

@@ -301,6 +301,13 @@ export const coreOperation: MigrateOperation = {
       {
         state: 'collaborative-content-to-storage',
         func: migrateCollaborativeContentToStorage
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_TX, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+          await client.update(DOMAIN_SPACE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },

models/document/src/migration.ts

@@ -228,6 +228,12 @@ export const documentOperation: MigrateOperation = {
       {
         state: 'renameFields',
         func: renameFields
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_DOCUMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },

models/drive/src/migration.ts

@@ -132,6 +132,12 @@ export const driveOperation: MigrateOperation = {
       {
         state: 'renameFields',
         func: renameFields
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_DRIVE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },

models/notification/src/migration.ts

@@ -393,6 +393,12 @@ export const notificationOperation: MigrateOperation = {
             { hidden: false }
           )
         }
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_DOC_NOTIFY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
 

models/task/src/migration.ts

@@ -584,6 +584,12 @@ export const taskOperation: MigrateOperation = {
         func: async (client: MigrationClient) => {
           await migrateSpace(client, task.space.Sequence, core.space.Workspace, [DOMAIN_KANBAN])
         }
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_TASK, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },

models/time/src/migration.ts

@@ -172,6 +172,12 @@ export const timeOperation: MigrateOperation = {
         func: async (client) => {
           await fillProps(client)
         }
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_TIME, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },

models/view/src/migration.ts

@@ -86,6 +86,12 @@ export const viewOperation: MigrateOperation = {
       {
         state: 'remove-done-state-filter',
         func: removeDoneStateFilter
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_VIEW, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },

server/mongo/src/storage.ts

@@ -251,6 +251,7 @@ abstract class MongoAdapterBase implements DbAdapter {
     operations: DocumentUpdate<T>
   ): Promise<void> {
     if (isOperator(operations)) {
+      await this.db.collection(domain).updateMany(this.translateRawQuery(query), { $set: { '%hash%': null } })
       await this.db
         .collection(domain)
         .updateMany(this.translateRawQuery(query), { ...operations } as unknown as UpdateFilter<Document>)