From f590002b946c80c4f97eab20bac77bd33679e86b Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 17 Apr 2023 21:53:19 +0700 Subject: [PATCH] TSK-1213: Allow to clean archived vacancies with content (#2999) --- dev/prod/package.json | 2 +- dev/tool/src/clean.ts | 42 +++++++++++++++++++++++++++ dev/tool/src/index.ts | 9 +++++- models/bitrix/package.json | 2 +- plugins/bitrix-assets/package.json | 2 +- plugins/bitrix-resources/package.json | 2 +- plugins/bitrix/package.json | 2 +- plugins/bitrix/src/sync.ts | 29 +++++++++++++++++- pods/server/package.json | 2 +- 9 files changed, 84 insertions(+), 8 deletions(-) diff --git a/dev/prod/package.json b/dev/prod/package.json index 25812af4b8..4c7197c984 100644 --- a/dev/prod/package.json +++ b/dev/prod/package.json @@ -157,7 +157,7 @@ "@hcengineering/document": "^0.6.0", "@hcengineering/document-assets": "^0.6.0", "@hcengineering/document-resources": "^0.6.0", - "@hcengineering/bitrix": "^0.6.30", + "@hcengineering/bitrix": "^0.6.31", "@hcengineering/bitrix-assets": "^0.6.0", "@hcengineering/bitrix-resources": "^0.6.0", "@hcengineering/request": "^0.6.0", diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts index 4f86f0e0d3..d15f3d7987 100644 --- a/dev/tool/src/clean.ts +++ b/dev/tool/src/clean.ts @@ -162,3 +162,45 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceId, transa await connection.close() } } +export async function cleanArchivedSpaces (workspaceId: WorkspaceId, transactorUrl: string): Promise { + const connection = (await connect(transactorUrl, workspaceId, undefined, { + mode: 'backup' + })) as unknown as CoreClient & BackupClient + try { + const count = 0 + const ops = new TxOperations(connection, core.account.System) + while (true) { + const spaces = await connection.findAll(core.class.Space, { archived: true }, { limit: 1000 }) + if (spaces.length === 0) { + break + } + + const h = connection.getHierarchy() + const withDomain = h + .getDescendants(core.class.Doc) + .filter((it) => h.findDomain(it) !== undefined) + .filter((it) => !h.isMixin(it)) + for (const c of withDomain) { + while (true) { + const docs = await connection.findAll(c, { space: { $in: spaces.map((it) => it._id) } }) + if (docs.length === 0) { + break + } + console.log('removing:', c, docs.length) + for (const d of docs) { + await ops.remove(d) + } + } + } + for (const s of spaces) { + await ops.remove(s) + } + } + + console.log('total docs with remove', count) + } catch (err: any) { + console.trace(err) + } finally { + await connection.close() + } +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 5a9273c3f9..0970fdc464 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -49,9 +49,9 @@ import { Data, getWorkspaceId, Tx, Version } from '@hcengineering/core' import { MinioService } from '@hcengineering/minio' import { MigrateOperation } from '@hcengineering/model' import { openAIConfigDefaults } from '@hcengineering/openai' +import { cleanArchivedSpaces, cleanRemovedTransactions, cleanWorkspace } from './clean' import { rebuildElastic } from './elastic' import { openAIConfig } from './openai' -import { cleanRemovedTransactions, cleanWorkspace } from './clean' /** * @public @@ -463,5 +463,12 @@ export function devTool ( await cleanRemovedTransactions(getWorkspaceId(workspace, productId), transactorUrl) }) + program + .command('clean-archived-spaces ') + .description('clean archived spaces') + .action(async (workspace: string, cmd: any) => { + await cleanArchivedSpaces(getWorkspaceId(workspace, productId), transactorUrl) + }) + program.parse(process.argv) } diff --git a/models/bitrix/package.json b/models/bitrix/package.json index f912e98f70..aa41c8cb2d 100644 --- a/models/bitrix/package.json +++ b/models/bitrix/package.json @@ -35,7 +35,7 @@ "@hcengineering/view": "^0.6.4", "@hcengineering/model-view": "^0.6.0", "@hcengineering/contact": "^0.6.13", - "@hcengineering/bitrix": "^0.6.30", + "@hcengineering/bitrix": "^0.6.31", "@hcengineering/bitrix-resources": "^0.6.0", "@hcengineering/preference": "^0.6.4", "@hcengineering/model-preference": "^0.6.0", diff --git a/plugins/bitrix-assets/package.json b/plugins/bitrix-assets/package.json index c9a068d385..577ac47a88 100644 --- a/plugins/bitrix-assets/package.json +++ b/plugins/bitrix-assets/package.json @@ -30,6 +30,6 @@ }, "dependencies": { "@hcengineering/platform": "^0.6.8", - "@hcengineering/bitrix": "^0.6.30" + "@hcengineering/bitrix": "^0.6.31" } } diff --git a/plugins/bitrix-resources/package.json b/plugins/bitrix-resources/package.json index 4f5276b504..2f5e27d6c3 100644 --- a/plugins/bitrix-resources/package.json +++ b/plugins/bitrix-resources/package.json @@ -34,7 +34,7 @@ "dependencies": { "@hcengineering/platform": "^0.6.8", "svelte": "3.55.1", - "@hcengineering/bitrix": "^0.6.30", + "@hcengineering/bitrix": "^0.6.31", "@hcengineering/ui": "^0.6.6", "@hcengineering/presentation": "^0.6.2", "@hcengineering/text-editor": "^0.6.0", diff --git a/plugins/bitrix/package.json b/plugins/bitrix/package.json index 5c1668b18f..ec6c3a867a 100644 --- a/plugins/bitrix/package.json +++ b/plugins/bitrix/package.json @@ -1,6 +1,6 @@ { "name": "@hcengineering/bitrix", - "version": "0.6.30", + "version": "0.6.31", "main": "lib/index.js", "author": "Anticrm Platform Contributors", "license": "EPL-2.0", diff --git a/plugins/bitrix/src/sync.ts b/plugins/bitrix/src/sync.ts index 7725a36d49..136e56ca8c 100644 --- a/plugins/bitrix/src/sync.ts +++ b/plugins/bitrix/src/sync.ts @@ -174,10 +174,30 @@ export async function syncDocument ( attachedTo: { $in: [resultDoc.document._id, ...attachIds] } }) for (const [ed, op, upd] of resultDoc.blobs) { - const existing = existingBlobs.find((it) => { + let existing = existingBlobs.find((it) => { const bdoc = hierarchy.as(it, bitrix.mixin.BitrixSyncDoc) return bdoc.bitrixId === ed.bitrixId }) + // Check attachment document exists in our storage. + + if (existing !== undefined) { + const ex = existing + try { + const resp = await fetch(concatLink(frontUrl, `/files?file=${existing?.file}&token=${info.token}`), { + method: 'GET' + }) + if (!resp.ok) { + // Attachment is broken and need to be re-added. + await applyOp.remove(ex) + existing = undefined + } + } catch (err: any) { + console.error(err) + await applyOp.remove(ex) + existing = undefined + } + } + // For Attachments, just do it once per attachment and assume it is not changed. if (existing === undefined) { const attachmentId: Ref = generateId() @@ -963,6 +983,13 @@ async function synchronizeUsers ( syncTime: Date.now() }) } + // TODO: Commented to replace names + // if (emp !== undefined && emp.name !== combineName(u.NAME, u.LAST_NAME)) { + // await ops.client.update(emp, { name: combineName(u.NAME, u.LAST_NAME) }) + // } + // if (account.name !== combineName(u.NAME, u.LAST_NAME)) { + // await ops.client.update(account, { name: combineName(u.NAME, u.LAST_NAME) }) + // } } userList.set(u.ID, accountId) } diff --git a/pods/server/package.json b/pods/server/package.json index 7c27943c78..09f43dab11 100644 --- a/pods/server/package.json +++ b/pods/server/package.json @@ -135,7 +135,7 @@ "@hcengineering/board": "^0.6.4", "@hcengineering/hr": "^0.6.4", "@hcengineering/document": "^0.6.0", - "@hcengineering/bitrix": "^0.6.30", + "@hcengineering/bitrix": "^0.6.31", "@hcengineering/request": "^0.6.0" } }