diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts index bbec65bbb3..1003a81635 100644 --- a/dev/tool/src/clean.ts +++ b/dev/tool/src/clean.ts @@ -14,13 +14,13 @@ // import { getAccountDB, listWorkspacesRaw } from '@hcengineering/account' -import attachment from '@hcengineering/attachment' import calendar from '@hcengineering/calendar' import chunter, { type ChatMessage } from '@hcengineering/chunter' import { loadCollaborativeDoc, saveCollaborativeDoc, yDocToBuffer } from '@hcengineering/collaboration' import contact from '@hcengineering/contact' import core, { type ArrOf, + type AttachedDoc, type BackupClient, type Class, ClassifierKind, @@ -45,11 +45,14 @@ import core, { SortingOrder, type Status, type StatusCategory, + type Tx, type TxCUD, + type TxCollectionCUD, type TxCreateDoc, type TxMixin, TxOperations, TxProcessor, + type TxRemoveDoc, type TxUpdateDoc, type WorkspaceId, generateId, @@ -93,30 +96,6 @@ export async function cleanWorkspace ( const hierarchy = ops.getHierarchy() - const attachments = await ops.findAll(attachment.class.Attachment, {}) - - const contacts = await ops.findAll(contact.class.Contact, {}) - - const files = new Set( - attachments.map((it) => it.file as string).concat(contacts.map((it) => it.avatar).filter((it) => it) as string[]) - ) - - const minioList = await storageAdapter.listStream(ctx, workspaceId) - const toClean: string[] = [] - while (true) { - const mvFiles = await minioList.next() - if (mvFiles.length === 0) { - break - } - - for (const mv of mvFiles) { - if (!files.has(mv._id)) { - toClean.push(mv._id) - } - } - } - await storageAdapter.remove(ctx, workspaceId, toClean) - if (opt.recruit) { const contacts = await ops.findAll(recruit.mixin.Candidate, {}) console.log('removing Talents', contacts.length) @@ -134,13 +113,6 @@ export async function cleanWorkspace ( const t2 = Date.now() console.log('remove time:', t2 - t, filter.length) } - - // const vacancies = await ops.findAll(recruit.class.Vacancy, {}) - // console.log('removing vacancies', vacancies.length) - // for (const c of vacancies) { - // console.log('Remove', c.name) - // await ops.remove(c) - // } } if (opt.tracker) { @@ -166,12 +138,39 @@ export async function cleanWorkspace ( const db = getWorkspaceMongoDB(_client, workspaceId) if (opt.removedTx) { - const txes = await db.collection(DOMAIN_TX).find({}).toArray() + let processed = 0 + const iterator = db.collection(DOMAIN_TX).find({}) + while (true) { + const txes: Tx[] = [] - for (const tx of txes) { - if (tx._class === core.class.TxRemoveDoc) { - // We need to remove all update and create operations for document - await db.collection(DOMAIN_TX).deleteMany({ objectId: tx.objectId }) + const doc = await iterator.next() + if (doc == null) { + break + } + txes.push(doc as unknown as Tx) + if (iterator.bufferedCount() > 0) { + txes.push(...(iterator.readBufferedDocuments() as unknown as Tx[])) + } + + for (const tx of txes) { + if (tx._class === core.class.TxRemoveDoc) { + // We need to remove all update and create operations for document + await db.collection(DOMAIN_TX).deleteMany({ objectId: (tx as TxRemoveDoc).objectId }) + processed++ + } + if ( + tx._class === core.class.TxCollectionCUD && + (tx as TxCollectionCUD).tx._class === core.class.TxRemoveDoc + ) { + // We need to remove all update and create operations for document + await db.collection(DOMAIN_TX).deleteMany({ + 'tx.objectId': ((tx as TxCollectionCUD).tx as TxRemoveDoc).objectId + }) + processed++ + } + } + if (processed % 1000 === 0) { + console.log('processed', processed) } } } diff --git a/models/activity/src/migration.ts b/models/activity/src/migration.ts index f504257d63..1f71a9b5f4 100644 --- a/models/activity/src/migration.ts +++ b/models/activity/src/migration.ts @@ -221,6 +221,12 @@ export const activityOperation: MigrateOperation = { { state: 'migrate-activity-markup', func: migrateActivityMarkup + }, + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_ACTIVITY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } } ]) }, diff --git a/models/attachment/src/migration.ts b/models/attachment/src/migration.ts index b820fd1b74..5fead855a8 100644 --- a/models/attachment/src/migration.ts +++ b/models/attachment/src/migration.ts @@ -13,9 +13,24 @@ // limitations under the License. // -import { type MigrateOperation, type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model' +import { + tryMigrate, + type MigrateOperation, + type MigrationClient, + type MigrationUpgradeClient +} from '@hcengineering/model' +import { attachmentId, DOMAIN_ATTACHMENT } from '.' export const attachmentOperation: MigrateOperation = { - async migrate (client: MigrationClient): Promise {}, + async migrate (client: MigrationClient): Promise { + await tryMigrate(client, attachmentId, [ + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_ATTACHMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } + } + ]) + }, async upgrade (state: Map>, client: () => Promise): Promise {} } diff --git a/models/chunter/src/migration.ts b/models/chunter/src/migration.ts index 70d2c2151c..65438a982b 100644 --- a/models/chunter/src/migration.ts +++ b/models/chunter/src/migration.ts @@ -362,6 +362,12 @@ export const chunterOperation: MigrateOperation = { 'attributeUpdates.attrKey': 'members' }) } + }, + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_CHUNTER, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } } ]) }, diff --git a/models/contact/src/migration.ts b/models/contact/src/migration.ts index 421bfb84f7..a109668aa8 100644 --- a/models/contact/src/migration.ts +++ b/models/contact/src/migration.ts @@ -1,5 +1,6 @@ // +import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact' import { type Class, type Doc, @@ -24,9 +25,8 @@ import { import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity' import core, { DOMAIN_SPACE } from '@hcengineering/model-core' import { DOMAIN_VIEW } from '@hcengineering/model-view' -import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact' -import contact, { contactId, DOMAIN_CONTACT } from './index' +import contact, { contactId, DOMAIN_CHANNEL, DOMAIN_CONTACT } from './index' async function createEmployeeEmail (client: TxOperations): Promise { const employees = await client.findAll(contact.mixin.Employee, {}) @@ -300,6 +300,13 @@ export const contactOperation: MigrateOperation = { { state: 'create-person-spaces-v1', func: createPersonSpaces + }, + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_CONTACT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + await client.update(DOMAIN_CHANNEL, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } } ]) }, diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 8baa7860f5..844d1e96f0 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -301,6 +301,13 @@ export const coreOperation: MigrateOperation = { { state: 'collaborative-content-to-storage', func: migrateCollaborativeContentToStorage + }, + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_TX, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + await client.update(DOMAIN_SPACE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } } ]) }, diff --git a/models/document/src/migration.ts b/models/document/src/migration.ts index 9b96bfd73e..84ed089391 100644 --- a/models/document/src/migration.ts +++ b/models/document/src/migration.ts @@ -228,6 +228,12 @@ export const documentOperation: MigrateOperation = { { state: 'renameFields', func: renameFields + }, + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_DOCUMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } } ]) }, diff --git a/models/drive/src/migration.ts b/models/drive/src/migration.ts index e2762c8b13..6ea69d90d2 100644 --- a/models/drive/src/migration.ts +++ b/models/drive/src/migration.ts @@ -132,6 +132,12 @@ export const driveOperation: MigrateOperation = { { state: 'renameFields', func: renameFields + }, + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_DRIVE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } } ]) }, diff --git a/models/notification/src/migration.ts b/models/notification/src/migration.ts index 7693cfb25d..4c67119729 100644 --- a/models/notification/src/migration.ts +++ b/models/notification/src/migration.ts @@ -393,6 +393,12 @@ export const notificationOperation: MigrateOperation = { { hidden: false } ) } + }, + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_DOC_NOTIFY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } } ]) diff --git a/models/task/src/migration.ts b/models/task/src/migration.ts index 93f7b7c0eb..a9f45ccacc 100644 --- a/models/task/src/migration.ts +++ b/models/task/src/migration.ts @@ -584,6 +584,12 @@ export const taskOperation: MigrateOperation = { func: async (client: MigrationClient) => { await migrateSpace(client, task.space.Sequence, core.space.Workspace, [DOMAIN_KANBAN]) } + }, + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_TASK, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } } ]) }, diff --git a/models/time/src/migration.ts b/models/time/src/migration.ts index 91e282515c..481e28dfe0 100644 --- a/models/time/src/migration.ts +++ b/models/time/src/migration.ts @@ -172,6 +172,12 @@ export const timeOperation: MigrateOperation = { func: async (client) => { await fillProps(client) } + }, + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_TIME, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } } ]) }, diff --git a/models/view/src/migration.ts b/models/view/src/migration.ts index f5587c0547..572a60d8d7 100644 --- a/models/view/src/migration.ts +++ b/models/view/src/migration.ts @@ -86,6 +86,12 @@ export const viewOperation: MigrateOperation = { { state: 'remove-done-state-filter', func: removeDoneStateFilter + }, + { + state: 'fix-rename-backups', + func: async (client: MigrationClient): Promise => { + await client.update(DOMAIN_VIEW, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + } } ]) }, diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index 1eacbf7e11..7e4f1cd848 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -251,6 +251,7 @@ abstract class MongoAdapterBase implements DbAdapter { operations: DocumentUpdate ): Promise { if (isOperator(operations)) { + await this.db.collection(domain).updateMany(this.translateRawQuery(query), { $set: { '%hash%': null } }) await this.db .collection(domain) .updateMany(this.translateRawQuery(query), { ...operations } as unknown as UpdateFilter)