From 18e9b76c616c8291c9f38cd4ed07a2cbb1a405a9 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Thu, 2 Mar 2023 01:38:39 +0700 Subject: [PATCH] TSK-734: Fix Bitrix email import (#2700) Signed-off-by: Andrey Sobolev --- plugins/activity-resources/src/utils.ts | 2 +- plugins/bitrix/src/sync.ts | 184 +++++++++++++++--------- plugins/bitrix/src/types.ts | 42 ++++-- plugins/bitrix/src/utils.ts | 43 +++++- 4 files changed, 183 insertions(+), 88 deletions(-) diff --git a/plugins/activity-resources/src/utils.ts b/plugins/activity-resources/src/utils.ts index 2605d982b2..e4fb7452e4 100644 --- a/plugins/activity-resources/src/utils.ts +++ b/plugins/activity-resources/src/utils.ts @@ -47,7 +47,7 @@ async function createPseudoViewlet ( } const docClass: Class = client.getModel().getObject(doc._class) - let trLabel = await translate(docClass.label, {}) + let trLabel = docClass.label !== undefined ? await translate(docClass.label, {}) : undefined if (dtx.collectionAttribute !== undefined) { const itemLabel = (dtx.collectionAttribute.type as Collection).itemLabel if (itemLabel !== undefined) { diff --git a/plugins/bitrix/src/sync.ts b/plugins/bitrix/src/sync.ts index 4e321d9e77..b267890c23 100644 --- a/plugins/bitrix/src/sync.ts +++ b/plugins/bitrix/src/sync.ts @@ -1,7 +1,8 @@ import attachment, { Attachment } from '@hcengineering/attachment' import chunter, { Comment } from '@hcengineering/chunter' -import contact, { combineName, Contact, EmployeeAccount } from '@hcengineering/contact' +import contact, { Channel, combineName, Contact, EmployeeAccount } from '@hcengineering/contact' import core, { + Account, AccountRole, ApplyOperations, AttachedDoc, @@ -18,10 +19,12 @@ import core, { MixinUpdate, Ref, Space, + Timestamp, TxOperations, TxProcessor, WithLookup } from '@hcengineering/core' +import gmail, { Message } from '@hcengineering/gmail' import tags, { TagElement } from '@hcengineering/tags' import { deepEqual } from 'fast-equals' import { BitrixClient } from './client' @@ -38,7 +41,7 @@ import { } from './types' import { convert, ConvertResult } from './utils' -async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data): Promise { +async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data, date: Timestamp): Promise { // We need to update fields if they are different. const documentUpdate: DocumentUpdate = {} for (const [k, v] of Object.entries(raw)) { @@ -51,7 +54,7 @@ async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data 0) { - await client.update(doc, documentUpdate) + await client.update(doc, documentUpdate, false, date, doc.modifiedBy) TxProcessor.applyUpdate(doc, documentUpdate) } return doc @@ -61,12 +64,14 @@ async function updateMixin ( client: ApplyOperations, doc: Doc, raw: Doc | Data, - mixin: Ref>> + mixin: Ref>>, + modifiedBy: Ref, + modifiedOn: Timestamp ): Promise { // We need to update fields if they are different. if (!client.getHierarchy().hasMixin(doc, mixin)) { - await client.createMixin(doc._id, doc._class, doc.space, mixin, raw as MixinData) + await client.createMixin(doc._id, doc._class, doc.space, mixin, raw as MixinData, modifiedOn, modifiedBy) return doc } @@ -81,7 +86,7 @@ async function updateMixin ( } } if (Object.keys(documentUpdate).length > 0) { - await client.updateMixin(doc._id, doc._class, doc.space, mixin, documentUpdate) + await client.updateMixin(doc._id, doc._class, doc.space, mixin, documentUpdate, modifiedOn, modifiedBy) } return doc } @@ -129,37 +134,26 @@ export async function syncDocument ( ) } - // Find all attachemnt documents to existing. + // Find all attachment documents to existing. const byClass = new Map>, (AttachedDoc & BitrixSyncDoc)[]>() + const idMapping = new Map, Ref>() for (const d of resultDoc.extraSync) { byClass.set(d._class, [...(byClass.get(d._class) ?? []), d]) } for (const [cl, vals] of byClass.entries()) { - if (applyOp.getHierarchy().isDerived(cl, core.class.AttachedDoc)) { - const existingByClass = await client.findAll(cl, { - attachedTo: resultDoc.document._id - }) + await syncClass(applyOp, cl, vals, idMapping, resultDoc.document._id) + } - for (const valValue of vals) { - const existingIdx = existingByClass.findIndex( - (it) => hierarchy.as(it, bitrix.mixin.BitrixSyncDoc).bitrixId === valValue.bitrixId - ) - // Update document id, for existing document. - valValue.attachedTo = resultDoc.document._id - let existing: Doc | undefined - if (existingIdx >= 0) { - existing = existingByClass.splice(existingIdx, 1).shift() - } - await updateAttachedDoc(existing, applyOp, valValue) - } - - // Remove previous merged documents, probable they are deleted in bitrix or wrongly migrated. - for (const doc of existingByClass) { - await client.remove(doc) - } - } + // Sync gmail documents + const emailAccount = resultDoc.extraSync.find( + (it) => + it._class === contact.class.Channel && (it as unknown as Channel).provider === contact.channelProvider.Email + ) + if (resultDoc.gmailDocuments.length > 0 && emailAccount !== undefined) { + const emailReadId = idMapping.get(emailAccount._id) ?? emailAccount._id + await syncClass(applyOp, gmail.class.Message, resultDoc.gmailDocuments, idMapping, emailReadId) } const existingBlobs = await client.findAll(attachment.class.Attachment, { @@ -229,6 +223,46 @@ export async function syncDocument ( } monitor?.(resultDoc) + async function syncClass ( + applyOp: ApplyOperations, + cl: Ref>, + vals: (AttachedDoc & BitrixSyncDoc)[], + idMapping: Map, Ref>, + attachedTo: Ref + ): Promise { + if (applyOp.getHierarchy().isDerived(cl, core.class.AttachedDoc)) { + const existingByClass = await client.findAll(cl, { + attachedTo + }) + + for (const valValue of vals) { + const id = idMapping.get(valValue.attachedTo) + if (id !== undefined) { + valValue.attachedTo = id + } else { + // Update document id, for existing document. + valValue.attachedTo = resultDoc.document._id + } + const existingIdx = existingByClass.findIndex( + (it) => hierarchy.as(it, bitrix.mixin.BitrixSyncDoc).bitrixId === valValue.bitrixId + ) + let existing: Doc | undefined + if (existingIdx >= 0) { + existing = existingByClass.splice(existingIdx, 1).shift() + if (existing !== undefined) { + idMapping.set(valValue._id, existing._id) + } + } + await updateAttachedDoc(existing, applyOp, valValue) + } + + // Remove previous merged documents, probable they are deleted in bitrix or wrongly migrated. + for (const doc of existingByClass) { + await applyOp.remove(doc) + } + } + } + async function updateAttachedDoc ( existing: WithLookup | undefined, applyOp: ApplyOperations, @@ -236,7 +270,7 @@ export async function syncDocument ( ): Promise { if (existing !== undefined) { // We need to update fields if they are different. - existing = await updateDoc(applyOp, existing, valValue) + existing = await updateDoc(applyOp, existing, valValue, Date.now()) const existingM = hierarchy.as(existing, bitrix.mixin.BitrixSyncDoc) await updateMixin( applyOp, @@ -246,7 +280,9 @@ export async function syncDocument ( bitrixId: valValue.bitrixId, rawData: valValue.rawData }, - bitrix.mixin.BitrixSyncDoc + bitrix.mixin.BitrixSyncDoc, + valValue.modifiedBy, + valValue.modifiedOn ) } else { const { bitrixId, rawData, ...data } = valValue @@ -283,7 +319,7 @@ export async function syncDocument ( // We need update doucment id. resultDoc.document._id = existing._id as Ref // We need to update fields if they are different. - return (await updateDoc(applyOp, existing, resultDoc.document)) as BitrixSyncDoc + return (await updateDoc(applyOp, existing, resultDoc.document, resultDoc.document.modifiedOn)) as BitrixSyncDoc // Go over extra documents. } else { const { bitrixId, rawData, ...data } = resultDoc.document @@ -323,7 +359,7 @@ async function updateMixins ( ) } else { const existingM = hierarchy.as(existing, mRef) - await updateMixin(applyOp, existingM, mv, mRef) + await updateMixin(applyOp, existingM, mv, mRef, resultDoc.modifiedBy, resultDoc.modifiedOn) } } } @@ -676,12 +712,13 @@ async function downloadComments ( order: { ID: ops.direction } }) for (const it of commentsData.result) { - const c: Comment & { bitrixId: string, type: string } = { + const c: Comment & BitrixSyncDoc = { _id: generateId(), _class: chunter.class.Comment, message: processComment(it.COMMENT as string), bitrixId: `${it.ID as string}`, type: it.ENTITY_TYPE, + rawData: it, attachedTo: res.document._id, attachedToClass: res.document._class, collection: 'comments', @@ -729,46 +766,51 @@ async function downloadComments ( } res.extraSync.push(c) } - const communications = await ops.bitrixClient.call('crm.activity.list', { - order: { ID: 'DESC' }, - filter: { - OWNER_ID: res.document.bitrixId, - OWNER_TYPE: ownerType.ID - }, - select: ['*', 'COMMUNICATIONS'] - }) - const cr = Array.isArray(communications.result) - ? (communications.result as BitrixActivity[]) - : [communications.result as BitrixActivity] - for (const comm of cr) { - const cummunications = comm.COMMUNICATIONS?.map((it) => it.ENTITY_SETTINGS?.LEAD_TITLE ?? '') - let message = `

- e-mail: ${cummunications?.join(',') ?? ''}
\n - Subject: ${comm.SUBJECT}
\n` - for (const [k, v] of Object.entries(comm.SETTINGS?.EMAIL_META ?? {}).concat( - Object.entries(comm.SETTINGS?.MESSAGE_HEADERS ?? {}) - )) { - if (v.trim().length > 0) { - message += `${k}: ${v}
\n` + const emailAccount = res.extraSync.find( + (it) => it._class === contact.class.Channel && (it as unknown as Channel).provider === contact.channelProvider.Email + ) + if (emailAccount !== undefined) { + const communications = await ops.bitrixClient.call('crm.activity.list', { + order: { ID: 'DESC' }, + filter: { + OWNER_ID: res.document.bitrixId, + OWNER_TYPE_ID: ownerType.ID + }, + select: ['*', 'COMMUNICATIONS'] + }) + const cr = Array.isArray(communications.result) + ? (communications.result as BitrixActivity[]) + : [communications.result as BitrixActivity] + for (const comm of cr) { + if (comm.PROVIDER_TYPE_ID === 'EMAIL') { + const parser = new DOMParser() + + const c: Message & BitrixSyncDoc = { + _id: generateId(), + _class: gmail.class.Message, + content: comm.DESCRIPTION, + textContent: + parser.parseFromString(comm.DESCRIPTION, 'text/html').textContent?.split('\n').slice(0, 3).join('\n') ?? '', + incoming: comm.DIRECTION === '1', + sendOn: new Date(comm.CREATED ?? new Date().toString()).getTime(), + subject: comm.SUBJECT, + bitrixId: `${comm.ID}`, + rawData: comm, + from: comm.SETTINGS?.EMAIL_META?.from ?? '', + to: comm.SETTINGS?.EMAIL_META?.to ?? '', + replyTo: comm.SETTINGS?.EMAIL_META?.replyTo ?? comm.SETTINGS?.MESSAGE_HEADERS?.['Reply-To'] ?? '', + messageId: comm.SETTINGS?.MESSAGE_HEADERS?.['Message-Id'] ?? '', + attachedTo: emailAccount._id as unknown as Ref, + attachedToClass: emailAccount._class, + collection: 'items', + space: res.document.space, + modifiedBy: userList.get(comm.AUTHOR_ID) ?? core.account.System, + modifiedOn: new Date(comm.CREATED ?? new Date().toString()).getTime() + } + res.gmailDocuments.push(c) } } - message += '

' + comm.DESCRIPTION - const c: Comment & { bitrixId: string, type: string } = { - _id: generateId(), - _class: chunter.class.Comment, - message, - bitrixId: `${comm.ID}`, - type: 'email', - attachedTo: res.document._id, - attachedToClass: res.document._class, - collection: 'comments', - space: res.document.space, - modifiedBy: userList.get(comm.AUTHOR_ID) ?? core.account.System, - modifiedOn: new Date(comm.CREATED ?? new Date().toString()).getTime() - } - - res.extraSync.push(c) } } diff --git a/plugins/bitrix/src/types.ts b/plugins/bitrix/src/types.ts index d803b3e8e9..f58cbe0bf4 100644 --- a/plugins/bitrix/src/types.ts +++ b/plugins/bitrix/src/types.ts @@ -266,25 +266,49 @@ export interface BitrixFieldMapping extends AttachedDoc { | FindReferenceOperation } +/** + * @public + */ +export interface BitrixCommunication { + ID: string + TYPE: 'EMAIL' | 'TASK' + VALUE: string // "a@gmail.com", + ENTITY_ID: string // "89013", + ENTITY_TYPE_ID: string // "1", + ENTITY_SETTINGS: { + HONORIFIC: string + NAME: string + SECOND_NAME: string + LAST_NAME: string + LEAD_TITLE: string + } +} + /** * @public */ export interface BitrixActivity { ID: string SUBJECT: string - COMMUNICATIONS?: { - ENTITY_SETTINGS?: { - LAST_NAME: string - NAME: string - LEAD_TITLE: string - } - }[] + PROVIDER_TYPE_ID: 'EMAIL' | 'TASK' + COMMUNICATIONS?: BitrixCommunication[] DESCRIPTION: string + DIRECTION: '1' | '2' AUTHOR_ID: string CREATED: number SETTINGS?: { - MESSAGE_HEADERS?: Record - EMAIL_META?: Record + MESSAGE_HEADERS?: Record & { + 'Message-Id': string // "", + 'Reply-To': string // "manager@a.com" + } + EMAIL_META?: Record & { + __email: string // some email + from: string // From email address + replyTo: string // ' + to: string // To email address + cc: string + bcc: string + } } } /** diff --git a/plugins/bitrix/src/utils.ts b/plugins/bitrix/src/utils.ts index ad9bdd6d13..287a7b5b6f 100644 --- a/plugins/bitrix/src/utils.ts +++ b/plugins/bitrix/src/utils.ts @@ -13,6 +13,7 @@ import core, { Space, WithLookup } from '@hcengineering/core' +import { Message } from '@hcengineering/gmail' import tags, { TagCategory, TagElement, TagReference } from '@hcengineering/tags' import bitrix, { BitrixEntityMapping, @@ -68,6 +69,7 @@ export interface ConvertResult { mixins: Record>, Data> // Mixins of document we will sync extraDocs: Doc[] // Extra documents we will sync, etc. extraSync: (AttachedDoc & BitrixSyncDoc)[] // Extra documents we will sync, etc. + gmailDocuments: (Message & BitrixSyncDoc)[] blobs: [Attachment & BitrixSyncDoc, () => Promise, (file: File, attach: Attachment) => void][] syncRequests: BitrixSyncRequest[] } @@ -134,8 +136,10 @@ export async function convert ( } return lval } else if (bfield.type === 'crm_multifield') { - if (Array.isArray(lval)) { - return lval.map((it) => it.VALUE) + if (lval != null && Array.isArray(lval)) { + return lval.map((it) => ({ value: it.VALUE, type: it.VALUE_TYPE.toLowerCase() })) + } else if (lval != null) { + return [{ value: lval.VALUE, type: lval.VALUE_TYPE.toLowerCase() }] } } else if (bfield.type === 'file') { if (Array.isArray(lval) && bfield.isMultiple) { @@ -236,14 +240,38 @@ export async function convert ( if (lval != null && lval !== '') { const vals = Array.isArray(lval) ? lval : [lval] for (const llVal of vals) { - const svalue = typeof llVal === 'string' ? llVal : `${JSON.stringify(llVal)}` - if (f.include != null || f.exclude != null) { - if (f.include !== undefined && svalue.match(f.include) == null) { + let svalue: string = typeof llVal === 'string' ? llVal : llVal.value + + if (typeof llVal === 'string') { + if (f.include != null || f.exclude != null) { + if (f.include !== undefined && svalue.match(f.include) == null) { + continue + } + if (f.exclude !== undefined && svalue.match(f.exclude) != null) { + continue + } + } + } else { + // TYPE matching to category. + if (f.provider === contact.channelProvider.Telegram && llVal.type !== 'telegram') { continue } - if (f.exclude !== undefined && svalue.match(f.exclude) != null) { + if (f.provider === contact.channelProvider.Whatsapp && llVal.type !== 'whatsapp') { continue } + if (f.provider === contact.channelProvider.Twitter && llVal.type !== 'twitter') { + continue + } + if (f.provider === contact.channelProvider.LinkedIn && llVal.type !== 'linkedin') { + continue + } + + // Fixes + if (f.provider === contact.channelProvider.Telegram) { + if (!svalue.startsWith('@') && !/^\d+/.test(svalue)) { + svalue = '@' + svalue + } + } } const c: Channel & BitrixSyncDoc = { _id: generateId(), @@ -452,7 +480,8 @@ export async function convert ( extraDocs: newExtraDocs, blobs, rawData: rawDocument, - syncRequests + syncRequests, + gmailDocuments: [] } }