From 2bbe9813fd0d78848ccbc3e9c4d0595967f36b3a Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@users.noreply.github.com>
Date: Mon, 14 Oct 2024 22:22:19 +0700
Subject: [PATCH] UBERF-8461: Fix migration do not clean backup info (#6913)
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
dev/tool/src/clean.ts | 73 ++++++++++++++--------------
models/activity/src/migration.ts | 6 +++
models/attachment/src/migration.ts | 19 +++++++-
models/chunter/src/migration.ts | 6 +++
models/contact/src/migration.ts | 11 ++++-
models/core/src/migration.ts | 7 +++
models/document/src/migration.ts | 6 +++
models/drive/src/migration.ts | 6 +++
models/notification/src/migration.ts | 6 +++
models/task/src/migration.ts | 6 +++
models/time/src/migration.ts | 6 +++
models/view/src/migration.ts | 6 +++
server/mongo/src/storage.ts | 1 +
13 files changed, 118 insertions(+), 41 deletions(-)
diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts
index bbec65bbb3..1003a81635 100644
--- a/dev/tool/src/clean.ts
+++ b/dev/tool/src/clean.ts
@@ -14,13 +14,13 @@
//
import { getAccountDB, listWorkspacesRaw } from '@hcengineering/account'
-import attachment from '@hcengineering/attachment'
import calendar from '@hcengineering/calendar'
import chunter, { type ChatMessage } from '@hcengineering/chunter'
import { loadCollaborativeDoc, saveCollaborativeDoc, yDocToBuffer } from '@hcengineering/collaboration'
import contact from '@hcengineering/contact'
import core, {
type ArrOf,
+ type AttachedDoc,
type BackupClient,
type Class,
ClassifierKind,
@@ -45,11 +45,14 @@ import core, {
SortingOrder,
type Status,
type StatusCategory,
+ type Tx,
type TxCUD,
+ type TxCollectionCUD,
type TxCreateDoc,
type TxMixin,
TxOperations,
TxProcessor,
+ type TxRemoveDoc,
type TxUpdateDoc,
type WorkspaceId,
generateId,
@@ -93,30 +96,6 @@ export async function cleanWorkspace (
const hierarchy = ops.getHierarchy()
- const attachments = await ops.findAll(attachment.class.Attachment, {})
-
- const contacts = await ops.findAll(contact.class.Contact, {})
-
- const files = new Set(
- attachments.map((it) => it.file as string).concat(contacts.map((it) => it.avatar).filter((it) => it) as string[])
- )
-
- const minioList = await storageAdapter.listStream(ctx, workspaceId)
- const toClean: string[] = []
- while (true) {
- const mvFiles = await minioList.next()
- if (mvFiles.length === 0) {
- break
- }
-
- for (const mv of mvFiles) {
- if (!files.has(mv._id)) {
- toClean.push(mv._id)
- }
- }
- }
- await storageAdapter.remove(ctx, workspaceId, toClean)
-
if (opt.recruit) {
const contacts = await ops.findAll(recruit.mixin.Candidate, {})
console.log('removing Talents', contacts.length)
@@ -134,13 +113,6 @@ export async function cleanWorkspace (
const t2 = Date.now()
console.log('remove time:', t2 - t, filter.length)
}
-
- // const vacancies = await ops.findAll(recruit.class.Vacancy, {})
- // console.log('removing vacancies', vacancies.length)
- // for (const c of vacancies) {
- // console.log('Remove', c.name)
- // await ops.remove(c)
- // }
}
if (opt.tracker) {
@@ -166,12 +138,39 @@ export async function cleanWorkspace (
const db = getWorkspaceMongoDB(_client, workspaceId)
if (opt.removedTx) {
- const txes = await db.collection(DOMAIN_TX).find({}).toArray()
+ let processed = 0
+ const iterator = db.collection(DOMAIN_TX).find({})
+ while (true) {
+ const txes: Tx[] = []
- for (const tx of txes) {
- if (tx._class === core.class.TxRemoveDoc) {
- // We need to remove all update and create operations for document
- await db.collection(DOMAIN_TX).deleteMany({ objectId: tx.objectId })
+ const doc = await iterator.next()
+ if (doc == null) {
+ break
+ }
+ txes.push(doc as unknown as Tx)
+ if (iterator.bufferedCount() > 0) {
+ txes.push(...(iterator.readBufferedDocuments() as unknown as Tx[]))
+ }
+
+ for (const tx of txes) {
+ if (tx._class === core.class.TxRemoveDoc) {
+ // We need to remove all update and create operations for document
+ await db.collection(DOMAIN_TX).deleteMany({ objectId: (tx as TxRemoveDoc<Doc>).objectId })
+ processed++
+ }
+ if (
+ tx._class === core.class.TxCollectionCUD &&
+ (tx as TxCollectionCUD<Doc, AttachedDoc>).tx._class === core.class.TxRemoveDoc
+ ) {
+ // We need to remove all update and create operations for document
+ await db.collection(DOMAIN_TX).deleteMany({
+ 'tx.objectId': ((tx as TxCollectionCUD<Doc, AttachedDoc>).tx as TxRemoveDoc<Doc>).objectId
+ })
+ processed++
+ }
+ }
+ if (processed % 1000 === 0) {
+ console.log('processed', processed)
}
}
}
diff --git a/models/activity/src/migration.ts b/models/activity/src/migration.ts
index f504257d63..1f71a9b5f4 100644
--- a/models/activity/src/migration.ts
+++ b/models/activity/src/migration.ts
@@ -221,6 +221,12 @@ export const activityOperation: MigrateOperation = {
{
state: 'migrate-activity-markup',
func: migrateActivityMarkup
+ },
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_ACTIVITY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
}
])
},
diff --git a/models/attachment/src/migration.ts b/models/attachment/src/migration.ts
index b820fd1b74..5fead855a8 100644
--- a/models/attachment/src/migration.ts
+++ b/models/attachment/src/migration.ts
@@ -13,9 +13,24 @@
// limitations under the License.
//
-import { type MigrateOperation, type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model'
+import {
+ tryMigrate,
+ type MigrateOperation,
+ type MigrationClient,
+ type MigrationUpgradeClient
+} from '@hcengineering/model'
+import { attachmentId, DOMAIN_ATTACHMENT } from '.'
export const attachmentOperation: MigrateOperation = {
- async migrate (client: MigrationClient): Promise<void> {},
+ async migrate (client: MigrationClient): Promise<void> {
+ await tryMigrate(client, attachmentId, [
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_ATTACHMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
+ }
+ ])
+ },
async upgrade (state: Map<string, Set<string>>, client: () => Promise<MigrationUpgradeClient>): Promise<void> {}
}
diff --git a/models/chunter/src/migration.ts b/models/chunter/src/migration.ts
index 70d2c2151c..65438a982b 100644
--- a/models/chunter/src/migration.ts
+++ b/models/chunter/src/migration.ts
@@ -362,6 +362,12 @@ export const chunterOperation: MigrateOperation = {
'attributeUpdates.attrKey': 'members'
})
}
+ },
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_CHUNTER, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
}
])
},
diff --git a/models/contact/src/migration.ts b/models/contact/src/migration.ts
index 421bfb84f7..a109668aa8 100644
--- a/models/contact/src/migration.ts
+++ b/models/contact/src/migration.ts
@@ -1,5 +1,6 @@
//
+import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact'
import {
type Class,
type Doc,
@@ -24,9 +25,8 @@ import {
import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity'
import core, { DOMAIN_SPACE } from '@hcengineering/model-core'
import { DOMAIN_VIEW } from '@hcengineering/model-view'
-import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact'
-import contact, { contactId, DOMAIN_CONTACT } from './index'
+import contact, { contactId, DOMAIN_CHANNEL, DOMAIN_CONTACT } from './index'
async function createEmployeeEmail (client: TxOperations): Promise<void> {
const employees = await client.findAll(contact.mixin.Employee, {})
@@ -300,6 +300,13 @@ export const contactOperation: MigrateOperation = {
{
state: 'create-person-spaces-v1',
func: createPersonSpaces
+ },
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_CONTACT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ await client.update(DOMAIN_CHANNEL, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
}
])
},
diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts
index 8baa7860f5..844d1e96f0 100644
--- a/models/core/src/migration.ts
+++ b/models/core/src/migration.ts
@@ -301,6 +301,13 @@ export const coreOperation: MigrateOperation = {
{
state: 'collaborative-content-to-storage',
func: migrateCollaborativeContentToStorage
+ },
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_TX, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ await client.update(DOMAIN_SPACE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
}
])
},
diff --git a/models/document/src/migration.ts b/models/document/src/migration.ts
index 9b96bfd73e..84ed089391 100644
--- a/models/document/src/migration.ts
+++ b/models/document/src/migration.ts
@@ -228,6 +228,12 @@ export const documentOperation: MigrateOperation = {
{
state: 'renameFields',
func: renameFields
+ },
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_DOCUMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
}
])
},
diff --git a/models/drive/src/migration.ts b/models/drive/src/migration.ts
index e2762c8b13..6ea69d90d2 100644
--- a/models/drive/src/migration.ts
+++ b/models/drive/src/migration.ts
@@ -132,6 +132,12 @@ export const driveOperation: MigrateOperation = {
{
state: 'renameFields',
func: renameFields
+ },
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_DRIVE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
}
])
},
diff --git a/models/notification/src/migration.ts b/models/notification/src/migration.ts
index 7693cfb25d..4c67119729 100644
--- a/models/notification/src/migration.ts
+++ b/models/notification/src/migration.ts
@@ -393,6 +393,12 @@ export const notificationOperation: MigrateOperation = {
{ hidden: false }
)
}
+ },
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_DOC_NOTIFY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
}
])
diff --git a/models/task/src/migration.ts b/models/task/src/migration.ts
index 93f7b7c0eb..a9f45ccacc 100644
--- a/models/task/src/migration.ts
+++ b/models/task/src/migration.ts
@@ -584,6 +584,12 @@ export const taskOperation: MigrateOperation = {
func: async (client: MigrationClient) => {
await migrateSpace(client, task.space.Sequence, core.space.Workspace, [DOMAIN_KANBAN])
}
+ },
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_TASK, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
}
])
},
diff --git a/models/time/src/migration.ts b/models/time/src/migration.ts
index 91e282515c..481e28dfe0 100644
--- a/models/time/src/migration.ts
+++ b/models/time/src/migration.ts
@@ -172,6 +172,12 @@ export const timeOperation: MigrateOperation = {
func: async (client) => {
await fillProps(client)
}
+ },
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_TIME, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
}
])
},
diff --git a/models/view/src/migration.ts b/models/view/src/migration.ts
index f5587c0547..572a60d8d7 100644
--- a/models/view/src/migration.ts
+++ b/models/view/src/migration.ts
@@ -86,6 +86,12 @@ export const viewOperation: MigrateOperation = {
{
state: 'remove-done-state-filter',
func: removeDoneStateFilter
+ },
+ {
+ state: 'fix-rename-backups',
+ func: async (client: MigrationClient): Promise<void> => {
+ await client.update(DOMAIN_VIEW, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+ }
}
])
},
diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts
index 1eacbf7e11..7e4f1cd848 100644
--- a/server/mongo/src/storage.ts
+++ b/server/mongo/src/storage.ts
@@ -251,6 +251,7 @@ abstract class MongoAdapterBase implements DbAdapter {
operations: DocumentUpdate<T>
): Promise<void> {
if (isOperator(operations)) {
+ await this.db.collection(domain).updateMany(this.translateRawQuery(query), { $set: { '%hash%': null } })
await this.db
.collection(domain)
.updateMany(this.translateRawQuery(query), { ...operations } as unknown as UpdateFilter<Document>)