Fix Bitrix attachments (#2683)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/client-resources/src/connection.ts

@@ -28,7 +28,7 @@ import {
   Ref,
   ServerStorage,
   Tx,
-  TxHander,
+  TxHandler,
   TxResult
 } from '@hcengineering/core'
 import { createInMemoryTxAdapter } from '@hcengineering/dev-storage'
@@ -45,7 +45,7 @@ import {
 
 class ServerStorageWrapper implements ClientConnection {
   measureCtx = new MeasureMetricsContext('client', {})
-  constructor (private readonly storage: ServerStorage, private readonly handler: TxHander) {}
+  constructor (private readonly storage: ServerStorage, private readonly handler: TxHandler) {}
 
   findAll<T extends Doc>(
     _class: Ref<Class<T>>,

packages/core/src/client.ts

@@ -28,7 +28,7 @@ import { toFindResult } from './utils'
 /**
  * @public
  */
-export type TxHander = (tx: Tx) => void
+export type TxHandler = (tx: Tx) => void
 
 /**
  * @public
@@ -146,7 +146,7 @@ class ClientImpl implements Client, BackupClient {
  * @public
  */
 export async function createClient (
-  connect: (txHandler: TxHander) => Promise<ClientConnection>,
+  connect: (txHandler: TxHandler) => Promise<ClientConnection>,
   // If set will build model with only allowed plugins.
   allowedPlugins?: Plugin[]
 ): Promise<Client> {
@@ -156,7 +156,7 @@ export async function createClient (
   const hierarchy = new Hierarchy()
   const model = new ModelDb(hierarchy)
 
-  function txHander (tx: Tx): void {
+  function txHandler (tx: Tx): void {
     if (client === null) {
       txBuffer?.push(tx)
     } else {
@@ -165,7 +165,7 @@ export async function createClient (
     }
   }
 
-  const conn = await connect(txHander)
+  const conn = await connect(txHandler)
   const atxes = await conn.findAll(
     core.class.Tx,
     { objectSpace: core.space.Model },
@@ -238,7 +238,7 @@ export async function createClient (
 
   client = new ClientImpl(hierarchy, model, conn)
 
-  for (const tx of txBuffer) txHander(tx)
+  for (const tx of txBuffer) txHandler(tx)
   txBuffer = undefined
 
   return client

plugins/bitrix-assets/lang/en.json

@@ -10,6 +10,7 @@
     "AddField": "Add mapping",
     "Attribute": "Attribute",
     "MapField": "Map...",
-    "BitrixImport": "Synchronize with Bitrix"
+    "BitrixImport": "Synchronize with Bitrix",
+    "AddMapping": "Add entity mapping"
   }
 }

plugins/bitrix-assets/lang/ru.json

@@ -10,6 +10,7 @@
     "AddField": "Add mapping",
     "Attribute": "Attribute",
     "MapField": "Map...",
-    "BitrixImport": "Synchronize with Bitrix"
+    "BitrixImport": "Synchronize with Bitrix",
+    "AddMapping": "Добавить отображение"
   }
 }

plugins/bitrix/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hcengineering/bitrix",
-  "version": "0.6.19",
+  "version": "0.6.23",
   "main": "lib/index.js",
   "author": "Anticrm Platform Contributors",
   "license": "EPL-2.0",

plugins/bitrix/src/sync.ts

@@ -14,6 +14,7 @@ import core, {
   generateId,
   Hierarchy,
   Mixin,
+  MixinData,
   MixinUpdate,
   Ref,
   Space,
@@ -45,7 +46,7 @@ async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data<Doc
       continue
     }
     const dv = (doc as any)[k]
-    if (!deepEqual(dv, v) && dv != null && v != null) {
+    if (!deepEqual(dv, v) && v != null) {
       ;(documentUpdate as any)[k] = v
     }
   }
@@ -63,13 +64,19 @@ async function updateMixin (
   mixin: Ref<Class<Mixin<Doc>>>
 ): Promise<Doc> {
   // We need to update fields if they are different.
+
+  if (!client.getHierarchy().hasMixin(doc, mixin)) {
+    await client.createMixin(doc._id, doc._class, doc.space, mixin, raw as MixinData<Doc, Doc>)
+    return doc
+  }
+
   const documentUpdate: MixinUpdate<Doc, Doc> = {}
   for (const [k, v] of Object.entries(raw)) {
     if (['_class', '_id', 'modifiedBy', 'modifiedOn', 'space', 'attachedTo', 'attachedToClass'].includes(k)) {
       continue
     }
     const dv = (doc as any)[k]
-    if (!deepEqual(dv, v) && dv != null && v != null) {
+    if (!deepEqual(dv, v) && v != null) {
       ;(documentUpdate as any)[k] = v
     }
   }
@@ -139,10 +146,13 @@ export async function syncDocument (
           const existingIdx = existingByClass.findIndex(
             (it) => hierarchy.as<Doc, BitrixSyncDoc>(it, bitrix.mixin.BitrixSyncDoc).bitrixId === valValue.bitrixId
           )
+          // Update document id, for existing document.
+          valValue.attachedTo = resultDoc.document._id
+          let existing: Doc | undefined
           if (existingIdx >= 0) {
-            const existing = existingByClass.splice(existingIdx, 1).shift()
-            await updateAttachedDoc(existing, applyOp, valValue)
+            existing = existingByClass.splice(existingIdx, 1).shift()
           }
+          await updateAttachedDoc(existing, applyOp, valValue)
         }
 
         // Remove previous merged documents, probable they are deleted in bitrix or wrongly migrated.
@@ -153,8 +163,7 @@ export async function syncDocument (
     }
 
     const existingBlobs = await client.findAll(attachment.class.Attachment, {
-      attachedTo: resultDoc.document._id,
-      [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: resultDoc.blobs.map((it) => it[0].bitrixId) }
+      attachedTo: resultDoc.document._id
     })
     for (const [ed, op, upd] of resultDoc.blobs) {
       const existing = existingBlobs.find(
@@ -171,33 +180,43 @@ export async function syncDocument (
           }
           const data = new FormData()
           data.append('file', edData)
-          const resp = await fetch(concatLink(frontUrl, '/files'), {
-            method: 'POST',
-            headers: {
-              Authorization: 'Bearer ' + info.token
-            },
-            body: data
-          })
-          if (resp.status === 200) {
-            const uuid = await resp.text()
-            upd(edData, ed)
-            await applyOp.addCollection(
-              ed._class,
-              ed.space,
-              ed.attachedTo,
-              ed.attachedToClass,
-              ed.collection,
-              {
-                file: uuid,
-                lastModified: edData.lastModified,
-                name: edData.name,
-                size: edData.size,
-                type: edData.type
+
+          upd(edData, ed)
+
+          ed.lastModified = edData.lastModified
+          ed.size = edData.size
+          ed.type = edData.type
+
+          let updated = false
+          for (const existingObj of existingBlobs) {
+            if (existingObj.name === ed.name && existingObj.size === ed.size && existingObj.type === ed.type) {
+              if (!updated) {
+                await updateAttachedDoc(existingObj, applyOp, ed)
+                updated = true
+              } else {
+                // Remove duplicate attachment
+                await applyOp.remove(existingObj)
+              }
+            }
+          }
+
+          if (!updated) {
+            // No attachment, send to server
+            const resp = await fetch(concatLink(frontUrl, '/files'), {
+              method: 'POST',
+              headers: {
+                Authorization: 'Bearer ' + info.token
               },
-              attachmentId,
-              ed.modifiedOn,
-              ed.modifiedBy
-            )
+              body: data
+            })
+            if (resp.status === 200) {
+              const uuid = await resp.text()
+
+              ed.file = uuid
+              ed._id = attachmentId as Ref<Attachment & BitrixSyncDoc>
+
+              await updateAttachedDoc(undefined, applyOp, ed)
+            }
           }
         } catch (err: any) {
           console.error(err)

plugins/bitrix/src/utils.ts

@@ -196,25 +196,22 @@ export async function convert (
 
     return r.join('').trim()
   }
-  const getDownloadValue = async (attr: AnyAttribute, operation: DownloadAttachmentOperation): Promise<any> => {
-    const r: Array<string | number | boolean | Date> = []
+  const getDownloadValue = async (
+    attr: AnyAttribute,
+    operation: DownloadAttachmentOperation
+  ): Promise<{ id: string, file: string }[]> => {
+    const files: Array<{ id: string, file: string }> = []
     for (const o of operation.fields) {
       const lval = extractValue(o.field)
       if (lval != null) {
         if (Array.isArray(lval)) {
-          r.push(...lval)
+          files.push(...lval)
         } else {
-          r.push(lval)
+          files.push(lval)
         }
       }
     }
-    if (r.length === 1) {
-      return r[0]
-    }
-    if (r.length === 0) {
-      return
-    }
-    return r.join('').trim()
+    return files
   }
 
   const getChannelValue = async (attr: AnyAttribute, operation: CreateChannelOperation): Promise<any> => {
@@ -332,8 +329,8 @@ export async function convert (
         value = await getTagValue(attr, f.operation)
         break
       case MappingOperation.DownloadAttachment: {
-        const blobRef: { file: string, id: string } = await getDownloadValue(attr, f.operation)
-        if (blobRef !== undefined) {
+        const blobRefs: { file: string, id: string }[] = await getDownloadValue(attr, f.operation)
+        for (const blobRef of blobRefs) {
           const attachDoc: Attachment & BitrixSyncDoc = {
             _id: generateId(),
             bitrixId: blobRef.id,
@@ -379,6 +376,7 @@ export async function convert (
             }
           ])
         }
+
         break
       }
     }

plugins/client-resources/src/connection.ts

@@ -26,7 +26,7 @@ import core, {
   FindResult,
   Ref,
   Tx,
-  TxHander,
+  TxHandler,
   TxResult
 } from '@hcengineering/core'
 import { getMetadata, PlatformError, readResponse, ReqId, serialize, UNAUTHORIZED } from '@hcengineering/platform'
@@ -51,7 +51,7 @@ class Connection implements ClientConnection {
 
   constructor (
     private readonly url: string,
-    private readonly handler: TxHander,
+    private readonly handler: TxHandler,
     private readonly onUpgrade?: () => void,
     private readonly onUnauthorized?: () => void
   ) {
@@ -229,7 +229,7 @@ class Connection implements ClientConnection {
  */
 export async function connect (
   url: string,
-  handler: TxHander,
+  handler: TxHandler,
   onUpgrade?: () => void,
   onUnauthorized?: () => void
 ): Promise<ClientConnection> {

plugins/client-resources/src/index.ts

@@ -14,7 +14,7 @@
 //
 
 import clientPlugin from '@hcengineering/client'
-import { Client, createClient, TxHander } from '@hcengineering/core'
+import { Client, createClient, TxHandler } from '@hcengineering/core'
 import { getMetadata, getPlugins, getResource } from '@hcengineering/platform'
 import { connect } from './connection'
 
@@ -49,7 +49,7 @@ export default async () => {
         if (client === undefined) {
           const filterModel = getMetadata(clientPlugin.metadata.FilterModel) ?? false
           client = createClient(
-            (handler: TxHander) => {
+            (handler: TxHandler) => {
               const url = new URL(`/${token}`, endpoint)
               console.log('connecting to', url.href)
               return connect(url.href, handler, onUpgrade, onUnauthorized)

server/mongo/src/storage.ts

@@ -603,10 +603,9 @@ class MongoAdapter extends MongoAdapterBase {
       if (lastDomain === undefined || bulkOperations.length === 0) {
         return
       }
+      const ops = bulkOperations.reduce<AnyBulkWriteOperation[]>((ops, op) => ops.concat(...(op.bulk ?? [])), [])
       try {
-        await this.db
-          .collection(lastDomain)
-          .bulkWrite(bulkOperations.reduce<AnyBulkWriteOperation[]>((ops, op) => ops.concat(...(op.bulk ?? [])), []))
+        await this.db.collection(lastDomain).bulkWrite(ops)
       } catch (err: any) {
         console.trace(err)
         throw err
@@ -913,6 +912,9 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter {
   txColl: Collection | undefined
 
   override async tx (...tx: Tx[]): Promise<TxResult> {
+    if (tx.length === 0) {
+      return {}
+    }
     await this.txCollection().insertMany(tx.map((it) => translateDoc(it)))
     return {}
   }