From fe6ee5e99f04ec5c695b54860ef8e100adef1b09 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Thu, 2 Feb 2023 19:08:09 +0700 Subject: [PATCH] Bitrix mass import (#2581) Signed-off-by: Andrey Sobolev --- packages/core/src/operations.ts | 12 +- packages/core/src/tx.ts | 11 +- .../src/components/AttributeMapper.svelte | 3 +- .../components/FieldMappingPresenter.svelte | 17 +- plugins/bitrix/package.json | 2 +- plugins/bitrix/src/sync.ts | 711 ++++++++++-------- plugins/bitrix/src/types.ts | 5 +- plugins/bitrix/src/utils.ts | 170 +++-- 8 files changed, 557 insertions(+), 374 deletions(-) diff --git a/packages/core/src/operations.ts b/packages/core/src/operations.ts index d6212fe3d7..702b7c1262 100644 --- a/packages/core/src/operations.ts +++ b/packages/core/src/operations.ts @@ -80,7 +80,9 @@ export class TxOperations implements Omit { attachedTo, space, collection, - this.txFactory.createTxCreateDoc

(_class, space, attributes as unknown as Data

, id, modifiedOn, modifiedBy) + this.txFactory.createTxCreateDoc

(_class, space, attributes as unknown as Data

, id, modifiedOn, modifiedBy), + modifiedOn, + modifiedBy ) await this.client.tx(tx) return tx.tx.objectId as unknown as Ref

@@ -103,7 +105,9 @@ export class TxOperations implements Omit { attachedTo, space, collection, - this.txFactory.createTxUpdateDoc(_class, space, objectId, operations, retrieve, modifiedOn, modifiedBy) + this.txFactory.createTxUpdateDoc(_class, space, objectId, operations, retrieve, modifiedOn, modifiedBy), + modifiedOn, + modifiedBy ) await this.client.tx(tx) return tx.objectId @@ -124,7 +128,9 @@ export class TxOperations implements Omit { attachedTo, space, collection, - this.txFactory.createTxRemoveDoc(_class, space, objectId, modifiedOn, modifiedBy) + this.txFactory.createTxRemoveDoc(_class, space, objectId, modifiedOn, modifiedBy), + modifiedOn, + modifiedBy ) await this.client.tx(tx) return tx.objectId diff --git a/packages/core/src/tx.ts b/packages/core/src/tx.ts index 047cb12283..311dd29bd1 100644 --- a/packages/core/src/tx.ts +++ b/packages/core/src/tx.ts @@ -304,7 +304,13 @@ export abstract class TxProcessor implements WithTx { static updateDoc2Doc(rawDoc: T, tx: TxUpdateDoc): T { const doc = _toDoc(rawDoc) - const ops = tx.operations as any + TxProcessor.applyUpdate(doc, tx.operations as any) + doc.modifiedBy = tx.modifiedBy + doc.modifiedOn = tx.modifiedOn + return rawDoc + } + + static applyUpdate(doc: T, ops: any): void { for (const key in ops) { if (key.startsWith('$')) { const operator = _getOperator(key) @@ -313,9 +319,6 @@ export abstract class TxProcessor implements WithTx { setObjectValue(key, doc, ops[key]) } } - doc.modifiedBy = tx.modifiedBy - doc.modifiedOn = tx.modifiedOn - return rawDoc } static updateMixin4Doc(rawDoc: D, tx: TxMixin): D { diff --git a/plugins/bitrix-resources/src/components/AttributeMapper.svelte b/plugins/bitrix-resources/src/components/AttributeMapper.svelte index 5ad0b58f2b..6d36f6c969 100644 --- a/plugins/bitrix-resources/src/components/AttributeMapper.svelte +++ b/plugins/bitrix-resources/src/components/AttributeMapper.svelte @@ -22,7 +22,8 @@ core.class.TypeDate, core.class.TypeNumber, core.class.EnumOf, - core.class.Collection + core.class.Collection, + core.class.ArrOf ]) function addMapping (evt: MouseEvent, kind: MappingOperation): void { diff --git a/plugins/bitrix-resources/src/components/FieldMappingPresenter.svelte b/plugins/bitrix-resources/src/components/FieldMappingPresenter.svelte index 2c4fd054bd..b3a3936579 100644 --- a/plugins/bitrix-resources/src/components/FieldMappingPresenter.svelte +++ b/plugins/bitrix-resources/src/components/FieldMappingPresenter.svelte @@ -2,7 +2,7 @@ import { BitrixEntityMapping, BitrixFieldMapping, MappingOperation } from '@hcengineering/bitrix' import { AnyAttribute } from '@hcengineering/core' import { getClient } from '@hcengineering/presentation' - import { Icon, IconArrowLeft, Label } from '@hcengineering/ui' + import { Button, Icon, IconArrowLeft, IconClose, Label } from '@hcengineering/ui' import CopyMappingPresenter from './mappings/CopyMappingPresenter.svelte' import CreateChannelMappingPresenter from './mappings/CreateChannelMappingPresenter.svelte' import CreateTagMappingPresenter from './mappings/CreateTagMappingPresenter.svelte' @@ -12,7 +12,12 @@ export let value: BitrixFieldMapping $: kind = value.operation.kind - const attr: AnyAttribute | undefined = getClient().getHierarchy().getAttribute(value.ofClass, value.attributeName) + let attr: AnyAttribute | undefined + try { + attr = getClient().getHierarchy().getAttribute(value.ofClass, value.attributeName) + } catch (err: any) { + console.error(err) + }

@@ -33,5 +38,13 @@ {:else if kind === MappingOperation.DownloadAttachment} {/if} + +
diff --git a/plugins/bitrix/package.json b/plugins/bitrix/package.json index 7b4f78d125..e075e0dd50 100644 --- a/plugins/bitrix/package.json +++ b/plugins/bitrix/package.json @@ -1,6 +1,6 @@ { "name": "@hcengineering/bitrix", - "version": "0.6.10", + "version": "0.6.15", "main": "lib/index.js", "author": "Anticrm Platform Contributors", "license": "EPL-2.0", diff --git a/plugins/bitrix/src/sync.ts b/plugins/bitrix/src/sync.ts index 97f23172a6..9a9843838d 100644 --- a/plugins/bitrix/src/sync.ts +++ b/plugins/bitrix/src/sync.ts @@ -10,16 +10,21 @@ import core, { Data, Doc, DocumentUpdate, + FindResult, generateId, + Hierarchy, Mixin, + MixinUpdate, Ref, Space, TxOperations, + TxProcessor, WithLookup } from '@hcengineering/core' import tags, { TagElement } from '@hcengineering/tags' import { deepEqual } from 'fast-equals' import { BitrixClient } from './client' +import bitrix from './index' import { BitrixActivity, BitrixEntityMapping, @@ -29,127 +34,134 @@ import { LoginInfo } from './types' import { convert, ConvertResult } from './utils' -import bitrix from './index' -async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data): Promise { +async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data): Promise { // We need to update fields if they are different. const documentUpdate: DocumentUpdate = {} for (const [k, v] of Object.entries(raw)) { - if (['_class', '_id', 'modifiedBy', 'modifiedOn', 'space'].includes(k)) { + if (['_class', '_id', 'modifiedBy', 'modifiedOn', 'space', 'attachedTo', 'attachedToClass'].includes(k)) { continue } - if (!deepEqual((doc as any)[k], v)) { + const dv = (doc as any)[k] + if (!deepEqual(dv, v) && dv != null && v != null) { ;(documentUpdate as any)[k] = v } } if (Object.keys(documentUpdate).length > 0) { await client.update(doc, documentUpdate) + TxProcessor.applyUpdate(doc, documentUpdate) } + return doc +} + +async function updateMixin ( + client: ApplyOperations, + doc: Doc, + raw: Doc | Data, + mixin: Ref>> +): Promise { + // We need to update fields if they are different. + const documentUpdate: MixinUpdate = {} + for (const [k, v] of Object.entries(raw)) { + if (['_class', '_id', 'modifiedBy', 'modifiedOn', 'space', 'attachedTo', 'attachedToClass'].includes(k)) { + continue + } + const dv = (doc as any)[k] + if (!deepEqual(dv, v) && dv != null && v != null) { + ;(documentUpdate as any)[k] = v + } + } + if (Object.keys(documentUpdate).length > 0) { + await client.updateMixin(doc._id, doc._class, doc.space, mixin, documentUpdate) + } + return doc } /** * @public */ -export async function syncPlatform ( +export async function syncDocument ( client: TxOperations, - mapping: BitrixEntityMapping, - documents: ConvertResult[], + existing: Doc | undefined, + resultDoc: ConvertResult, info: LoginInfo, frontUrl: string, monitor?: (doc: ConvertResult) => void ): Promise { - const existingDocuments = await client.findAll(mapping.ofClass, { - [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: documents.map((it) => it.document.bitrixId) } - }) const hierarchy = client.getHierarchy() - let syncronized = 0 - for (const d of documents) { - try { - const existing = existingDocuments.find( - (it) => hierarchy.as(it, bitrix.mixin.BitrixSyncDoc).bitrixId === d.document.bitrixId + + try { + const applyOp = client.apply('bitrix') + // const newDoc = existing === undefined + existing = await updateMainDoc(applyOp) + + const mixins = { ...resultDoc.mixins } + + // Add bitrix sync mixin + mixins[bitrix.mixin.BitrixSyncDoc] = { + type: resultDoc.document.type, + bitrixId: resultDoc.document.bitrixId, + rawData: resultDoc.rawData, + syncTime: Date.now() + } + + // Check and update mixins + await updateMixins(mixins, hierarchy, existing, applyOp, resultDoc.document) + + // Just create supplier documents, like TagElements. + for (const ed of resultDoc.extraDocs) { + await applyOp.createDoc( + ed._class, + ed.space, + ed, + ed._id, + resultDoc.document.modifiedOn, + resultDoc.document.modifiedBy ) - const applyOp = client.apply('bitrix') - if (existing !== undefined) { - // We need update doucment id. - d.document._id = existing._id as Ref - // We need to update fields if they are different. - await updateDoc(applyOp, existing, d.document) + } - // Check and update mixins - for (const [m, mv] of Object.entries(d.mixins)) { - const mRef = m as Ref> - if (hierarchy.hasMixin(existing, mRef)) { - await applyOp.createMixin( - d.document._id, - d.document._class, - d.document.space, - m as Ref>, - mv, - d.document.modifiedOn, - d.document.modifiedBy - ) - } else { - const existingM = hierarchy.as(existing, mRef) - await updateDoc(applyOp, existingM, mv) - } - } - } else { - await applyOp.createDoc( - d.document._class, - d.document.space, - d.document, - d.document._id, - d.document.modifiedOn, - d.document.modifiedBy - ) + // Find all attachemnt documents to existing. + const byClass = new Map>, (AttachedDoc & BitrixSyncDoc)[]>() - await applyOp.createMixin( - d.document._id, - d.document._class, - d.document.space, - bitrix.mixin.BitrixSyncDoc, - { - type: d.document.type, - bitrixId: d.document.bitrixId - }, - d.document.modifiedOn, - d.document.modifiedBy - ) - for (const [m, mv] of Object.entries(d.mixins)) { - await applyOp.createMixin( - d.document._id, - d.document._class, - d.document.space, - m as Ref>, - mv, - d.document.modifiedOn, - d.document.modifiedBy + for (const d of resultDoc.extraSync) { + byClass.set(d._class, [...(byClass.get(d._class) ?? []), d]) + } + + for (const [cl, vals] of byClass.entries()) { + if (applyOp.getHierarchy().isDerived(cl, core.class.AttachedDoc)) { + const existingByClass = await client.findAll(cl, { + attachedTo: resultDoc.document._id, + [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: vals.map((it) => it.bitrixId) } + }) + + for (const valValue of vals) { + const existing = existingByClass.find( + (it) => hierarchy.as(it, bitrix.mixin.BitrixSyncDoc).bitrixId === valValue.bitrixId ) + await updateAttachedDoc(existing, applyOp, valValue) } - for (const ed of d.extraDocs) { - if (applyOp.getHierarchy().isDerived(ed._class, core.class.AttachedDoc)) { - const adoc = ed as AttachedDoc - await applyOp.addCollection( - adoc._class, - adoc.space, - adoc.attachedTo, - adoc.attachedToClass, - adoc.collection, - adoc, - adoc._id, - d.document.modifiedOn, - d.document.modifiedBy - ) - } else { - await applyOp.createDoc(ed._class, ed.space, ed, ed._id, d.document.modifiedOn, d.document.modifiedBy) + } + } + + const existingBlobs = await client.findAll(attachment.class.Attachment, { + attachedTo: resultDoc.document._id, + [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: resultDoc.blobs.map((it) => it[0].bitrixId) } + }) + for (const [ed, op] of resultDoc.blobs) { + const existing = existingBlobs.find( + (it) => hierarchy.as(it, bitrix.mixin.BitrixSyncDoc).bitrixId === ed.bitrixId + ) + // For Attachments, just do it once per attachment and assume it is not changed. + if (existing === undefined) { + const attachmentId: Ref = generateId() + try { + const edData = await op() + if (edData === undefined) { + console.error('Failed to retrieve document data', ed.name) + continue } - } - - for (const ed of d.blobs) { - const attachmentId: Ref = generateId() - const data = new FormData() - data.append('file', ed) + data.append('file', edData) const resp = await fetch(concatLink(frontUrl, '/files'), { method: 'POST', headers: { @@ -162,76 +174,129 @@ export async function syncPlatform ( await applyOp.addCollection( attachment.class.Attachment, - d.document.space, - d.document._id, - d.document._class, + resultDoc.document.space, + resultDoc.document._id, + resultDoc.document._class, 'attachments', { file: uuid, - lastModified: ed.lastModified, - name: ed.name, - size: ed.size, - type: ed.type + lastModified: edData.lastModified, + name: edData.name, + size: edData.size, + type: edData.type }, attachmentId, - d.document.modifiedOn, - d.document.modifiedBy + resultDoc.document.modifiedOn, + resultDoc.document.modifiedBy ) } - } - - if (d.comments !== undefined) { - const comments = await d.comments - if (comments !== undefined && comments.length > 0) { - const existingComments = await client.findAll(chunter.class.Comment, { - attachedTo: d.document._id, - [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: comments.map((it) => it.bitrixId) } - }) - - for (const comment of comments) { - const existing = existingComments.find( - (it) => hierarchy.as(it, bitrix.mixin.BitrixSyncDoc).bitrixId === comment.bitrixId - ) - if (existing !== undefined) { - // We need to update fields if they are different. - await updateDoc(applyOp, existing, comment) - } else { - await applyOp.addCollection( - comment._class, - comment.space, - comment.attachedTo, - comment.attachedToClass, - comment.collection, - comment, - comment._id, - comment.modifiedOn, - comment.modifiedBy - ) - - await applyOp.createMixin( - comment._id, - comment._class, - comment.space, - bitrix.mixin.BitrixSyncDoc, - { - type: d.document.type, - bitrixId: d.document.bitrixId - }, - comment.modifiedOn, - comment.modifiedBy - ) - } - } - } + } catch (err: any) { + console.error(err) } } - await applyOp.commit() - } catch (err: any) { - console.error(err) } - console.log('SYNC:', syncronized, documents.length - syncronized) - syncronized++ - monitor?.(d) + await applyOp.commit() + } catch (err: any) { + console.error(err) + } + monitor?.(resultDoc) + + async function updateAttachedDoc ( + existing: WithLookup | undefined, + applyOp: ApplyOperations, + valValue: AttachedDoc & BitrixSyncDoc + ): Promise { + if (existing !== undefined) { + // We need to update fields if they are different. + existing = await updateDoc(applyOp, existing, valValue) + const existingM = hierarchy.as(existing, bitrix.mixin.BitrixSyncDoc) + await updateMixin( + applyOp, + existingM, + { + type: valValue.type, + bitrixId: valValue.bitrixId, + rawData: valValue.rawData + }, + bitrix.mixin.BitrixSyncDoc + ) + } else { + const { bitrixId, rawData, ...data } = valValue + await applyOp.addCollection( + valValue._class, + valValue.space, + valValue.attachedTo, + valValue.attachedToClass, + valValue.collection, + data, + valValue._id, + valValue.modifiedOn, + valValue.modifiedBy + ) + + await applyOp.createMixin( + valValue._id, + valValue._class, + valValue.space, + bitrix.mixin.BitrixSyncDoc, + { + type: valValue.type, + bitrixId: valValue.bitrixId, + rawData: valValue.rawData + }, + valValue.modifiedOn, + valValue.modifiedBy + ) + } + } + + async function updateMainDoc (applyOp: ApplyOperations): Promise { + if (existing !== undefined) { + // We need update doucment id. + resultDoc.document._id = existing._id as Ref + // We need to update fields if they are different. + return (await updateDoc(applyOp, existing, resultDoc.document)) as BitrixSyncDoc + // Go over extra documents. + } else { + const { bitrixId, rawData, ...data } = resultDoc.document + const id = await applyOp.createDoc( + resultDoc.document._class, + resultDoc.document.space, + data, + resultDoc.document._id, + resultDoc.document.modifiedOn, + resultDoc.document.modifiedBy + ) + resultDoc.document._id = id as Ref + + return resultDoc.document + } + } +} + +async function updateMixins ( + mixins: Record>, Data>, + hierarchy: Hierarchy, + existing: Doc, + applyOp: ApplyOperations, + resultDoc: BitrixSyncDoc +): Promise { + for (const [m, mv] of Object.entries(mixins)) { + const mRef = m as Ref> + if (!hierarchy.hasMixin(existing, mRef)) { + await applyOp.createMixin( + resultDoc._id, + resultDoc._class, + resultDoc.space, + m as Ref>, + mv, + resultDoc.modifiedOn, + resultDoc.modifiedBy + ) + } else { + const existingM = hierarchy.as(existing, mRef) + await updateMixin(applyOp, existingM, mv, mRef) + } } } @@ -288,6 +353,8 @@ export function processComment (comment: string): string { return comment } +// 1 day +const syncPeriod = 1000 * 60 * 60 * 24 /** * @public */ @@ -312,6 +379,200 @@ export async function performSynchronization (ops: { const userList = new Map>() // Fill all users and create new ones, if required. + await synchronizeUsers(userList, ops, allEmployee) + + try { + if (ops.space === undefined || ops.mapping.$lookup?.fields === undefined) { + return + } + let processed = 0 + + let added = 0 + + const sel = ['*', 'UF_*'] + if (ops.mapping.type === BitrixEntityType.Lead) { + sel.push('EMAIL') + sel.push('IM') + } + + const allTagElements = await ops.client.findAll(tags.class.TagElement, {}) + + while (added < ops.limit) { + const result = await ops.bitrixClient.call(ops.mapping.type + '.list', { + select: sel, + order: { ID: ops.direction }, + start: processed + }) + + const fields = ops.mapping.$lookup?.fields as BitrixFieldMapping[] + + const toProcess = result.result as any[] + const syncTime = Date.now() + + const existingDocuments = await ops.client.findAll(ops.mapping.ofClass, { + [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: toProcess.map((it) => `${it.ID as string}`) } + }) + const defaultCategories = await ops.client.findAll(tags.class.TagCategory, { + default: true + }) + let synchronized = 0 + while (toProcess.length > 0) { + console.log('LOAD:', synchronized, added) + synchronized++ + const [r] = toProcess.slice(0, 1) + + const existingDoc = existingDocuments.find( + (it) => ops.client.getHierarchy().as(it, bitrix.mixin.BitrixSyncDoc).bitrixId === r.ID + ) + if (existingDoc !== undefined) { + const bd = ops.client.getHierarchy().as(existingDoc, bitrix.mixin.BitrixSyncDoc) + if (bd.syncTime !== undefined && bd.syncTime + syncPeriod > syncTime) { + // No need to sync, sime sync time is not yet arrived. + toProcess.splice(0, 1) + added++ + ops.monitor?.(result.total) + if (added >= ops.limit) { + break + } + continue + } + } + // Convert documents. + try { + const res = await convert( + ops.client, + ops.mapping, + ops.space, + fields, + r, + userList, + existingDoc, + defaultCategories, + allTagElements, + ops.blobProvider + ) + + if (ops.mapping.comments) { + await downloadComments(res, ops, commentFieldKeys, userList) + } + + added++ + const total = result.total + await syncDocument(ops.client, existingDoc, res, ops.loginInfo, ops.frontUrl, () => { + ops.monitor?.(total) + }) + for (const d of res.extraDocs) { + // update tags if required + if (d._class === tags.class.TagElement) { + allTagElements.push(d as TagElement) + } + } + if (added >= ops.limit) { + break + } + } catch (err: any) { + console.log('failed to obtain data for', r, err) + await new Promise((resolve) => { + // Sleep for a while + setTimeout(resolve, 1000) + }) + } + toProcess.splice(0, 1) + } + + processed = result.next + } + } catch (err: any) { + console.error(err) + } +} +async function downloadComments ( + res: ConvertResult, + ops: { + client: TxOperations + bitrixClient: BitrixClient + space: Ref | undefined + mapping: WithLookup + limit: number + direction: 'ASC' | 'DSC' + frontUrl: string + loginInfo: LoginInfo + monitor: (total: number) => void + blobProvider?: ((blobRef: any) => Promise) | undefined + }, + commentFieldKeys: string[], + userList: Map> +): Promise { + const commentsData = await ops.bitrixClient.call(BitrixEntityType.Comment + '.list', { + filter: { + ENTITY_ID: res.document.bitrixId, + ENTITY_TYPE: ops.mapping.type.replace('crm.', '') + }, + select: commentFieldKeys, + order: { ID: ops.direction } + }) + for (const it of commentsData.result) { + const c: Comment & { bitrixId: string, type: string } = { + _id: generateId(), + _class: chunter.class.Comment, + message: processComment(it.COMMENT as string), + bitrixId: it.ID, + type: it.ENTITY_TYPE, + attachedTo: res.document._id, + attachedToClass: res.document._class, + collection: 'comments', + space: res.document.space, + modifiedBy: userList.get(it.AUTHOR_ID) ?? core.account.System, + modifiedOn: new Date(it.CREATED ?? new Date().toString()).getTime() + } + res.extraSync.push(c) + } + const communications = await ops.bitrixClient.call('crm.activity.list', { + order: { ID: 'DESC' }, + filter: { + OWNER_ID: res.document.bitrixId + }, + select: ['*', 'COMMUNICATIONS'] + }) + const cr = Array.isArray(communications.result) + ? (communications.result as BitrixActivity[]) + : [communications.result as BitrixActivity] + for (const comm of cr) { + const c: Comment & { bitrixId: string, type: string } = { + _id: generateId(), + _class: chunter.class.Comment, + message: `e-mail:
+ Subject: ${comm.SUBJECT} + ${comm.DESCRIPTION}`, + bitrixId: comm.ID, + type: 'email', + attachedTo: res.document._id, + attachedToClass: res.document._class, + collection: 'comments', + space: res.document.space, + modifiedBy: userList.get(comm.AUTHOR_ID) ?? core.account.System, + modifiedOn: new Date(comm.CREATED ?? new Date().toString()).getTime() + } + res.extraSync.push(c) + } +} + +async function synchronizeUsers ( + userList: Map>, + ops: { + client: TxOperations + bitrixClient: BitrixClient + space: Ref | undefined + mapping: WithLookup + limit: number + direction: 'ASC' | 'DSC' + frontUrl: string + loginInfo: LoginInfo + monitor: (total: number) => void + blobProvider?: ((blobRef: any) => Promise) | undefined + }, + allEmployee: FindResult +): Promise { let totalUsers = 1 let next = 0 while (userList.size < totalUsers) { @@ -337,152 +598,4 @@ export async function performSynchronization (ops: { userList.set(u.ID, accountId) } } - - try { - if (ops.space === undefined || ops.mapping.$lookup?.fields === undefined) { - return - } - let processed = 0 - const tagElements: Map>, TagElement[]> = new Map() - - let added = 0 - - while (added < ops.limit) { - const sel = ['*', 'UF_*'] - if (ops.mapping.type === BitrixEntityType.Lead) { - sel.push('EMAIL') - sel.push('IM') - } - const result = await ops.bitrixClient.call(ops.mapping.type + '.list', { - select: sel, - order: { ID: ops.direction }, - start: processed - }) - - const extraDocs: Doc[] = [] - - const fields = ops.mapping.$lookup?.fields as BitrixFieldMapping[] - - const toProcess = result.result as any[] - - const existingDocuments = await ops.client.findAll( - ops.mapping.ofClass, - { - [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: toProcess.map((it) => `${it.ID as string}`) } - }, - { - projection: { - _id: 1, - [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: 1 - } - } - ) - const defaultCategories = await ops.client.findAll(tags.class.TagCategory, { - default: true - }) - let synchronized = 0 - while (toProcess.length > 0) { - const convertResults: ConvertResult[] = [] - - console.log('LOAD:', synchronized, added) - synchronized++ - const [r] = toProcess.slice(0, 1) - // Convert documents. - try { - const res = await convert( - ops.client, - ops.mapping, - ops.space, - fields, - r, - extraDocs, - tagElements, - userList, - existingDocuments, - defaultCategories, - ops.blobProvider - ) - if (ops.mapping.comments) { - res.comments = await ops.bitrixClient - .call(BitrixEntityType.Comment + '.list', { - filter: { - ENTITY_ID: res.document.bitrixId, - ENTITY_TYPE: ops.mapping.type.replace('crm.', '') - }, - select: commentFieldKeys, - order: { ID: ops.direction } - }) - .then((comments) => { - return comments.result.map((it: any) => { - const c: Comment & { bitrixId: string, type: string } = { - _id: generateId(), - _class: chunter.class.Comment, - message: processComment(it.COMMENT as string), - bitrixId: it.ID, - type: it.ENTITY_TYPE, - attachedTo: res.document._id, - attachedToClass: res.document._class, - collection: 'comments', - space: res.document.space, - modifiedBy: userList.get(it.AUTHOR_ID) ?? core.account.System, - modifiedOn: new Date(it.CREATED ?? new Date().toString()).getTime() - } - return c - }) - }) - const communications = await ops.bitrixClient.call('crm.activity.list', { - order: { ID: 'DESC' }, - filter: { - OWNER_ID: res.document.bitrixId - }, - select: ['*', 'COMMUNICATIONS'] - }) - const cr = Array.isArray(communications.result) - ? (communications.result as BitrixActivity[]) - : [communications.result as BitrixActivity] - for (const comm of cr) { - const c: Comment & { bitrixId: string, type: string } = { - _id: generateId(), - _class: chunter.class.Comment, - message: `e-mail:
- Subject: ${comm.SUBJECT} - ${comm.DESCRIPTION}`, - bitrixId: comm.ID, - type: 'email', - attachedTo: res.document._id, - attachedToClass: res.document._class, - collection: 'comments', - space: res.document.space, - modifiedBy: userList.get(comm.AUTHOR_ID) ?? core.account.System, - modifiedOn: new Date(comm.CREATED ?? new Date().toString()).getTime() - } - res.comments?.push(c) - } - } - - convertResults.push(res) - extraDocs.push(...res.extraDocs) - added++ - const total = result.total - await syncPlatform(ops.client, ops.mapping, convertResults, ops.loginInfo, ops.frontUrl, () => { - ops.monitor?.(total) - }) - if (added >= ops.limit) { - break - } - } catch (err: any) { - console.log('failed to obtain data for', r, err) - await new Promise((resolve) => { - // Sleep for a while - setTimeout(resolve, 1000) - }) - } - toProcess.splice(0, 1) - } - - processed = result.next - } - } catch (err: any) { - console.error(err) - } } diff --git a/plugins/bitrix/src/types.ts b/plugins/bitrix/src/types.ts index f6898bc48e..f1bbf59cf5 100644 --- a/plugins/bitrix/src/types.ts +++ b/plugins/bitrix/src/types.ts @@ -86,8 +86,11 @@ export interface LoginInfo { * @public */ export interface BitrixSyncDoc extends Doc { - type: string + type?: string bitrixId: string + syncTime?: number + // raw bitrix document data. + rawData?: any } /** diff --git a/plugins/bitrix/src/utils.ts b/plugins/bitrix/src/utils.ts index 836dd50b2a..60c1d773e9 100644 --- a/plugins/bitrix/src/utils.ts +++ b/plugins/bitrix/src/utils.ts @@ -1,6 +1,18 @@ -import { Comment } from '@hcengineering/chunter' +import attachment, { Attachment } from '@hcengineering/attachment' import contact, { Channel, EmployeeAccount } from '@hcengineering/contact' -import core, { AnyAttribute, Class, Client, Data, Doc, generateId, Mixin, Ref, Space } from '@hcengineering/core' +import core, { + AnyAttribute, + AttachedDoc, + Class, + Client, + Data, + Doc, + generateId, + Mixin, + Ref, + Space, + WithLookup +} from '@hcengineering/core' import tags, { TagCategory, TagElement, TagReference } from '@hcengineering/tags' import { BitrixEntityMapping, @@ -12,7 +24,6 @@ import { DownloadAttachmentOperation, MappingOperation } from '.' -import bitrix from './index' /** * @public @@ -42,10 +53,11 @@ export function collectFields (fieldMapping: BitrixFieldMapping[]): string[] { */ export interface ConvertResult { document: BitrixSyncDoc // Document we should achive + rawData: any mixins: Record>, Data> // Mixins of document we will achive - extraDocs: Doc[] // Extra documents we will achive, comments etc. - blobs: File[] // - comments?: Array + extraDocs: Doc[] // Extra documents we will achive, etc. + extraSync: (AttachedDoc & BitrixSyncDoc)[] // Extra documents we will achive, etc. + blobs: [Attachment & BitrixSyncDoc, () => Promise][] } /** @@ -57,11 +69,10 @@ export async function convert ( space: Ref, fields: BitrixFieldMapping[], rawDocument: any, - prevExtra: Doc[], // <<-- a list of previous extra documents, so for example TagElement will be reused, if present for more what one item and required to be created - tagElements: Map>, TagElement[]>, // TagElement cache. userList: Map>, - existingDocuments: Doc[], + existingDoc: WithLookup | undefined, defaultCategories: TagCategory[], + allTagElements: TagElement[], blobProvider?: (blobRef: any) => Promise ): Promise { const hierarchy = client.getHierarchy() @@ -76,13 +87,13 @@ export async function convert ( modifiedBy: userList.get(rawDocument.CREATED_BY_ID) ?? core.account.System } - const existingId = existingDocuments.find((it) => (it as any)[bitrix.mixin.BitrixSyncDoc].bitrixId === bitrixId) - ?._id as Ref + const existingId = existingDoc?._id // Obtain a proper modified by for document + const newExtraSyncDocs: (AttachedDoc & BitrixSyncDoc)[] = [] const newExtraDocs: Doc[] = [] - const blobs: File[] = [] + const blobs: [Attachment & BitrixSyncDoc, () => Promise][] = [] const mixins: Record>, Data> = {} const extractValue = (field?: string, alternatives?: string[]): any | undefined => { @@ -119,18 +130,32 @@ export async function convert ( return lval } else if (bfield.type === 'date') { if (lval !== '' && lval != null) { - return new Date(lval) + return new Date(lval).getTime() } } else if (bfield.type === 'char') { return lval === 'Y' } else if (bfield.type === 'enumeration' || bfield.type === 'crm_status') { if (lval != null && lval !== '') { - if (bfield.isMultiple && Array.isArray(lval)) { - lval = lval[0] ?? '' - } - const eValue = bfield.items?.find((it) => it.ID === lval)?.VALUE - if (eValue !== undefined) { - return eValue + if (bfield.isMultiple) { + const results: any[] = [] + for (let llval of Array.isArray(lval) ? lval : [lval]) { + if (typeof llval === 'number') { + llval = llval.toString() + } + const eValue = bfield.items?.find((it) => it.ID === llval)?.VALUE + if (eValue !== undefined) { + results.push(eValue) + } + } + return results + } else { + if (typeof lval === 'number') { + lval = lval.toString() + } + const eValue = bfield.items?.find((it) => it.ID === lval)?.VALUE + if (eValue !== undefined) { + return eValue + } } } } @@ -145,15 +170,24 @@ export async function convert ( } const lval = extractValue(o.field, o.alternatives) if (lval != null) { - r.push(lval) + if (Array.isArray(lval)) { + r.push(...lval) + } else { + r.push(lval) + } } } - if (r.length === 1) { - return r[0] - } if (r.length === 0) { return } + + if (attr.type._class === core.class.ArrOf) { + return r + } + if (r.length === 1) { + return r[0] + } + return r.join('').trim() } const getDownloadValue = async (attr: AnyAttribute, operation: DownloadAttachmentOperation): Promise => { @@ -192,7 +226,7 @@ export async function convert ( continue } } - const c: Channel = { + const c: Channel & BitrixSyncDoc = { _id: generateId(), _class: contact.class.Channel, attachedTo: document._id, @@ -202,9 +236,10 @@ export async function convert ( value: svalue, provider: f.provider, space: document.space, - modifiedOn: document.modifiedOn + modifiedOn: document.modifiedOn, + bitrixId: svalue } - newExtraDocs.push(c) + newExtraSyncDocs.push(c) } } } @@ -212,22 +247,6 @@ export async function convert ( } const getTagValue = async (attr: AnyAttribute, operation: CreateTagOperation): Promise => { - const elements = - tagElements.get(attr.attributeOf) ?? - (await client.findAll(tags.class.TagElement, { - targetClass: attr.attributeOf - })) - - const references = - existingId !== undefined - ? await client.findAll(tags.class.TagReference, { - attachedTo: existingId - }) - : [] - // Add tags creation requests from previous conversions. - elements.push(...prevExtra.filter((it) => it._class === tags.class.TagElement).map((it) => it as TagElement)) - - tagElements.set(attr.attributeOf, elements) const defaultCategory = defaultCategories.find((it) => it.targetClass === attr.attributeOf) if (defaultCategory === undefined) { @@ -245,13 +264,14 @@ export async function convert ( } else { vals = [lval as string] } + let ci = 0 for (let vv of vals) { vv = vv.trim() if (vv === '') { continue } // Find existing element and create reference based on it. - let tag: TagElement | undefined = elements.find((it) => it.title === vv) + let tag: TagElement | undefined = allTagElements.find((it) => it.title === vv) if (tag === undefined) { tag = { _id: generateId(), @@ -267,24 +287,22 @@ export async function convert ( } newExtraDocs.push(tag) } - const ref: TagReference = { + const ref: TagReference & BitrixSyncDoc = { _id: generateId(), attachedTo: existingId ?? document._id, attachedToClass: attr.attributeOf, collection: attr.name, _class: tags.class.TagReference, tag: tag._id, - color: 1, + color: ci++, title: vv, weight: o.weight, modifiedBy: document.modifiedBy, modifiedOn: document.modifiedOn, - space: tags.space.Tags - } - if (references.find((it) => it.title === vv) === undefined) { - // Add only if not already added - newExtraDocs.push(ref) + space: tags.space.Tags, + bitrixId: vv } + newExtraSyncDocs.push(ref) } } return undefined @@ -310,19 +328,45 @@ export async function convert ( case MappingOperation.DownloadAttachment: { const blobRef: { file: string, id: string } = await getDownloadValue(attr, f.operation) if (blobRef !== undefined) { - const response = await blobProvider?.(blobRef) - if (response !== undefined) { - let fname = blobRef.id - switch (response.type) { - case 'application/pdf': - fname += '.pdf' - break - case 'application/msword': - fname += '.doc' - break - } - blobs.push(new File([response], fname, { type: response.type })) + const attachDoc: Attachment & BitrixSyncDoc = { + _id: generateId(), + bitrixId: blobRef.id, + file: '', // Empty since not uploaded yet. + name: blobRef.id, + size: -1, + type: 'application/octet-stream', + lastModified: 0, + attachedTo: existingId ?? document._id, + attachedToClass: attr.attributeOf, + collection: attr.name, + _class: attachment.class.Attachment, + modifiedBy: document.modifiedBy, + modifiedOn: document.modifiedOn, + space: document.space } + + blobs.push([ + attachDoc, + async () => { + if (blobRef !== undefined) { + const response = await blobProvider?.(blobRef) + if (response !== undefined) { + let fname = blobRef.id + switch (response.type) { + case 'application/pdf': + fname += '.pdf' + break + case 'application/msword': + fname += '.doc' + break + } + attachDoc.type = response.type + attachDoc.name = fname + return new File([response], fname, { type: response.type }) + } + } + } + ]) } break } @@ -339,7 +383,7 @@ export async function convert ( } } - return { document, mixins, extraDocs: newExtraDocs, blobs } + return { document, mixins, extraSync: newExtraSyncDocs, extraDocs: newExtraDocs, blobs, rawData: rawDocument } } /**