From 96f0fef45075d000b5404d02a2f9411b91c22e50 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Fri, 24 Feb 2023 15:04:03 +0700 Subject: [PATCH] Fix Bitrix attachments (#2683) Signed-off-by: Andrey Sobolev --- dev/client-resources/src/connection.ts | 4 +- packages/core/src/client.ts | 10 +-- plugins/bitrix-assets/lang/en.json | 3 +- plugins/bitrix-assets/lang/ru.json | 3 +- plugins/bitrix/package.json | 2 +- plugins/bitrix/src/sync.ts | 83 +++++++++++++--------- plugins/bitrix/src/utils.ts | 24 +++---- plugins/client-resources/src/connection.ts | 6 +- plugins/client-resources/src/index.ts | 4 +- server/mongo/src/storage.ts | 8 ++- 10 files changed, 84 insertions(+), 63 deletions(-) diff --git a/dev/client-resources/src/connection.ts b/dev/client-resources/src/connection.ts index ee9cf218e4..99b31bda19 100644 --- a/dev/client-resources/src/connection.ts +++ b/dev/client-resources/src/connection.ts @@ -28,7 +28,7 @@ import { Ref, ServerStorage, Tx, - TxHander, + TxHandler, TxResult } from '@hcengineering/core' import { createInMemoryTxAdapter } from '@hcengineering/dev-storage' @@ -45,7 +45,7 @@ import { class ServerStorageWrapper implements ClientConnection { measureCtx = new MeasureMetricsContext('client', {}) - constructor (private readonly storage: ServerStorage, private readonly handler: TxHander) {} + constructor (private readonly storage: ServerStorage, private readonly handler: TxHandler) {} findAll( _class: Ref>, diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 4cb981e0f4..1f65235548 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -28,7 +28,7 @@ import { toFindResult } from './utils' /** * @public */ -export type TxHander = (tx: Tx) => void +export type TxHandler = (tx: Tx) => void /** * @public @@ -146,7 +146,7 @@ class ClientImpl implements Client, BackupClient { * @public */ export async function createClient ( - connect: (txHandler: TxHander) => Promise, + connect: (txHandler: TxHandler) => Promise, // If set will build model with only allowed plugins. allowedPlugins?: Plugin[] ): Promise { @@ -156,7 +156,7 @@ export async function createClient ( const hierarchy = new Hierarchy() const model = new ModelDb(hierarchy) - function txHander (tx: Tx): void { + function txHandler (tx: Tx): void { if (client === null) { txBuffer?.push(tx) } else { @@ -165,7 +165,7 @@ export async function createClient ( } } - const conn = await connect(txHander) + const conn = await connect(txHandler) const atxes = await conn.findAll( core.class.Tx, { objectSpace: core.space.Model }, @@ -238,7 +238,7 @@ export async function createClient ( client = new ClientImpl(hierarchy, model, conn) - for (const tx of txBuffer) txHander(tx) + for (const tx of txBuffer) txHandler(tx) txBuffer = undefined return client diff --git a/plugins/bitrix-assets/lang/en.json b/plugins/bitrix-assets/lang/en.json index 08c0a0955e..5a4cf1ae1a 100644 --- a/plugins/bitrix-assets/lang/en.json +++ b/plugins/bitrix-assets/lang/en.json @@ -10,6 +10,7 @@ "AddField": "Add mapping", "Attribute": "Attribute", "MapField": "Map...", - "BitrixImport": "Synchronize with Bitrix" + "BitrixImport": "Synchronize with Bitrix", + "AddMapping": "Add entity mapping" } } diff --git a/plugins/bitrix-assets/lang/ru.json b/plugins/bitrix-assets/lang/ru.json index 08c0a0955e..5ce20346c6 100644 --- a/plugins/bitrix-assets/lang/ru.json +++ b/plugins/bitrix-assets/lang/ru.json @@ -10,6 +10,7 @@ "AddField": "Add mapping", "Attribute": "Attribute", "MapField": "Map...", - "BitrixImport": "Synchronize with Bitrix" + "BitrixImport": "Synchronize with Bitrix", + "AddMapping": "Добавить отображение" } } diff --git a/plugins/bitrix/package.json b/plugins/bitrix/package.json index 2878450ceb..0202441e31 100644 --- a/plugins/bitrix/package.json +++ b/plugins/bitrix/package.json @@ -1,6 +1,6 @@ { "name": "@hcengineering/bitrix", - "version": "0.6.19", + "version": "0.6.23", "main": "lib/index.js", "author": "Anticrm Platform Contributors", "license": "EPL-2.0", diff --git a/plugins/bitrix/src/sync.ts b/plugins/bitrix/src/sync.ts index 3c57ce5704..36285bb439 100644 --- a/plugins/bitrix/src/sync.ts +++ b/plugins/bitrix/src/sync.ts @@ -14,6 +14,7 @@ import core, { generateId, Hierarchy, Mixin, + MixinData, MixinUpdate, Ref, Space, @@ -45,7 +46,7 @@ async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data>> ): Promise { // We need to update fields if they are different. + + if (!client.getHierarchy().hasMixin(doc, mixin)) { + await client.createMixin(doc._id, doc._class, doc.space, mixin, raw as MixinData) + return doc + } + const documentUpdate: MixinUpdate = {} for (const [k, v] of Object.entries(raw)) { if (['_class', '_id', 'modifiedBy', 'modifiedOn', 'space', 'attachedTo', 'attachedToClass'].includes(k)) { continue } const dv = (doc as any)[k] - if (!deepEqual(dv, v) && dv != null && v != null) { + if (!deepEqual(dv, v) && v != null) { ;(documentUpdate as any)[k] = v } } @@ -139,10 +146,13 @@ export async function syncDocument ( const existingIdx = existingByClass.findIndex( (it) => hierarchy.as(it, bitrix.mixin.BitrixSyncDoc).bitrixId === valValue.bitrixId ) + // Update document id, for existing document. + valValue.attachedTo = resultDoc.document._id + let existing: Doc | undefined if (existingIdx >= 0) { - const existing = existingByClass.splice(existingIdx, 1).shift() - await updateAttachedDoc(existing, applyOp, valValue) + existing = existingByClass.splice(existingIdx, 1).shift() } + await updateAttachedDoc(existing, applyOp, valValue) } // Remove previous merged documents, probable they are deleted in bitrix or wrongly migrated. @@ -153,8 +163,7 @@ export async function syncDocument ( } const existingBlobs = await client.findAll(attachment.class.Attachment, { - attachedTo: resultDoc.document._id, - [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: resultDoc.blobs.map((it) => it[0].bitrixId) } + attachedTo: resultDoc.document._id }) for (const [ed, op, upd] of resultDoc.blobs) { const existing = existingBlobs.find( @@ -171,33 +180,43 @@ export async function syncDocument ( } const data = new FormData() data.append('file', edData) - const resp = await fetch(concatLink(frontUrl, '/files'), { - method: 'POST', - headers: { - Authorization: 'Bearer ' + info.token - }, - body: data - }) - if (resp.status === 200) { - const uuid = await resp.text() - upd(edData, ed) - await applyOp.addCollection( - ed._class, - ed.space, - ed.attachedTo, - ed.attachedToClass, - ed.collection, - { - file: uuid, - lastModified: edData.lastModified, - name: edData.name, - size: edData.size, - type: edData.type + + upd(edData, ed) + + ed.lastModified = edData.lastModified + ed.size = edData.size + ed.type = edData.type + + let updated = false + for (const existingObj of existingBlobs) { + if (existingObj.name === ed.name && existingObj.size === ed.size && existingObj.type === ed.type) { + if (!updated) { + await updateAttachedDoc(existingObj, applyOp, ed) + updated = true + } else { + // Remove duplicate attachment + await applyOp.remove(existingObj) + } + } + } + + if (!updated) { + // No attachment, send to server + const resp = await fetch(concatLink(frontUrl, '/files'), { + method: 'POST', + headers: { + Authorization: 'Bearer ' + info.token }, - attachmentId, - ed.modifiedOn, - ed.modifiedBy - ) + body: data + }) + if (resp.status === 200) { + const uuid = await resp.text() + + ed.file = uuid + ed._id = attachmentId as Ref + + await updateAttachedDoc(undefined, applyOp, ed) + } } } catch (err: any) { console.error(err) diff --git a/plugins/bitrix/src/utils.ts b/plugins/bitrix/src/utils.ts index d6ceba73f2..0e10184da4 100644 --- a/plugins/bitrix/src/utils.ts +++ b/plugins/bitrix/src/utils.ts @@ -196,25 +196,22 @@ export async function convert ( return r.join('').trim() } - const getDownloadValue = async (attr: AnyAttribute, operation: DownloadAttachmentOperation): Promise => { - const r: Array = [] + const getDownloadValue = async ( + attr: AnyAttribute, + operation: DownloadAttachmentOperation + ): Promise<{ id: string, file: string }[]> => { + const files: Array<{ id: string, file: string }> = [] for (const o of operation.fields) { const lval = extractValue(o.field) if (lval != null) { if (Array.isArray(lval)) { - r.push(...lval) + files.push(...lval) } else { - r.push(lval) + files.push(lval) } } } - if (r.length === 1) { - return r[0] - } - if (r.length === 0) { - return - } - return r.join('').trim() + return files } const getChannelValue = async (attr: AnyAttribute, operation: CreateChannelOperation): Promise => { @@ -332,8 +329,8 @@ export async function convert ( value = await getTagValue(attr, f.operation) break case MappingOperation.DownloadAttachment: { - const blobRef: { file: string, id: string } = await getDownloadValue(attr, f.operation) - if (blobRef !== undefined) { + const blobRefs: { file: string, id: string }[] = await getDownloadValue(attr, f.operation) + for (const blobRef of blobRefs) { const attachDoc: Attachment & BitrixSyncDoc = { _id: generateId(), bitrixId: blobRef.id, @@ -379,6 +376,7 @@ export async function convert ( } ]) } + break } } diff --git a/plugins/client-resources/src/connection.ts b/plugins/client-resources/src/connection.ts index 80792a4200..422f54ad13 100644 --- a/plugins/client-resources/src/connection.ts +++ b/plugins/client-resources/src/connection.ts @@ -26,7 +26,7 @@ import core, { FindResult, Ref, Tx, - TxHander, + TxHandler, TxResult } from '@hcengineering/core' import { getMetadata, PlatformError, readResponse, ReqId, serialize, UNAUTHORIZED } from '@hcengineering/platform' @@ -51,7 +51,7 @@ class Connection implements ClientConnection { constructor ( private readonly url: string, - private readonly handler: TxHander, + private readonly handler: TxHandler, private readonly onUpgrade?: () => void, private readonly onUnauthorized?: () => void ) { @@ -229,7 +229,7 @@ class Connection implements ClientConnection { */ export async function connect ( url: string, - handler: TxHander, + handler: TxHandler, onUpgrade?: () => void, onUnauthorized?: () => void ): Promise { diff --git a/plugins/client-resources/src/index.ts b/plugins/client-resources/src/index.ts index 204540deef..88f2e99293 100644 --- a/plugins/client-resources/src/index.ts +++ b/plugins/client-resources/src/index.ts @@ -14,7 +14,7 @@ // import clientPlugin from '@hcengineering/client' -import { Client, createClient, TxHander } from '@hcengineering/core' +import { Client, createClient, TxHandler } from '@hcengineering/core' import { getMetadata, getPlugins, getResource } from '@hcengineering/platform' import { connect } from './connection' @@ -49,7 +49,7 @@ export default async () => { if (client === undefined) { const filterModel = getMetadata(clientPlugin.metadata.FilterModel) ?? false client = createClient( - (handler: TxHander) => { + (handler: TxHandler) => { const url = new URL(`/${token}`, endpoint) console.log('connecting to', url.href) return connect(url.href, handler, onUpgrade, onUnauthorized) diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index a2fb477485..80118979bd 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -603,10 +603,9 @@ class MongoAdapter extends MongoAdapterBase { if (lastDomain === undefined || bulkOperations.length === 0) { return } + const ops = bulkOperations.reduce((ops, op) => ops.concat(...(op.bulk ?? [])), []) try { - await this.db - .collection(lastDomain) - .bulkWrite(bulkOperations.reduce((ops, op) => ops.concat(...(op.bulk ?? [])), [])) + await this.db.collection(lastDomain).bulkWrite(ops) } catch (err: any) { console.trace(err) throw err @@ -913,6 +912,9 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter { txColl: Collection | undefined override async tx (...tx: Tx[]): Promise { + if (tx.length === 0) { + return {} + } await this.txCollection().insertMany(tx.map((it) => translateDoc(it))) return {} }