UBERF-10408: Use new messages

Signed-off-by: Artem Savchenko <armisav@gmail.com>

models/mail/src/index.ts

@@ -54,6 +54,7 @@ export function createModel (builder: Builder): void {
     chat.icon.Thread,
     mail.string.MailThread,
     mail.string.MailMessages,
+    undefined,
     chat.masterTag.Thread
   )
   createSystemType(
@@ -62,6 +63,7 @@ export function createModel (builder: Builder): void {
     chat.icon.Channel,
     mail.string.MailChannel,
     mail.string.MailChannels,
+    undefined,
     chat.masterTag.Channel
   )
   createMailViewlet(builder)

services/gmail/pod-gmail-next/package.json

@@ -17,7 +17,7 @@
     "_phase:bundle": "rushx bundle",
     "_phase:docker-build": "rushx docker:build",
     "_phase:docker-staging": "rushx docker:staging",
-    "bundle": "node ../../../common/scripts/esbuild.js",
+    "bundle": "node ../../../common/scripts/esbuild.js --external=ws",
     "docker:build": "../../../common/scripts/docker_build.sh hardcoreeng/gmail",
     "docker:tbuild": "docker build -t hardcoreeng/gmail . --platform=linux/amd64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/gmail",
     "docker:staging": "../../../common/scripts/docker_tag.sh hardcoreeng/gmail staging",
@@ -60,6 +60,7 @@
     "@hcengineering/account-client": "^0.6.0",
     "@hcengineering/api-client": "^0.6.0",
     "@hcengineering/card": "^0.6.0",
+    "@hcengineering/chat": "^0.6.0",
     "@hcengineering/client": "^0.6.18",
     "@hcengineering/client-resources": "^0.6.27",
     "@hcengineering/communication-rest-client": "0.1.172",

services/gmail/pod-gmail-next/src/__tests__/attachments.test.ts

@@ -83,9 +83,10 @@ describe('AttachmentHandler', () => {
   describe('addAttachement', () => {
     it('should not add attachment if it already exists', async () => {
       const file: AttachedFile = {
-        file: 'test-file',
+        id: 'test-id',
+        data: Buffer.from('test-file'),
         name: 'test.txt',
-        type: 'text/plain',
+        contentType: 'text/plain',
         size: 100,
         lastModified: Date.now()
       }
@@ -106,9 +107,10 @@ describe('AttachmentHandler', () => {
 
     it('should add new attachment', async () => {
       const file: AttachedFile = {
-        file: 'test-file',
+        id: 'test-id',
+        data: Buffer.from('test-file'),
         name: 'test.txt',
-        type: 'text/plain',
+        contentType: 'text/plain',
         size: 100,
         lastModified: Date.now()
       }
@@ -154,9 +156,10 @@ describe('AttachmentHandler', () => {
 
       expect(result).toEqual([
         {
-          file: 'test-data',
+          id: expect.any(String),
+          data: expect.any(Buffer),
           name: 'test.txt',
-          type: 'text/plain',
+          contentType: 'text/plain',
           size: 100,
           lastModified: expect.any(Number)
         }
@@ -177,9 +180,10 @@ describe('AttachmentHandler', () => {
 
       expect(result).toEqual([
         {
-          file: 'test-data',
+          id: expect.any(String),
+          data: expect.any(Buffer),
           name: 'test.txt',
-          type: 'text/plain',
+          contentType: 'text/plain',
           size: 100,
           lastModified: expect.any(Number)
         }

services/gmail/pod-gmail-next/src/__tests__/message.test.ts

@@ -1,4 +1,4 @@
-import { type MeasureContext, PersonId, TxOperations, AttachedData } from '@hcengineering/core'
+import { type MeasureContext, TxOperations, AttachedData } from '@hcengineering/core'
 import { type GaxiosResponse } from 'gaxios'
 import { gmail_v1 } from 'googleapis'
 import { type Message } from '@hcengineering/gmail'
@@ -17,7 +17,7 @@ describe('MessageManager', () => {
   let mockCtx: MeasureContext
   let mockClient: TxOperations
   let mockAttachmentHandler: AttachmentHandler
-  let mockSocialId: PersonId
+  let mockToken: string
   let mockWorkspace: { getChannel: (email: string) => Channel | undefined }
 
   beforeEach(() => {
@@ -37,13 +37,13 @@ describe('MessageManager', () => {
       addAttachement: jest.fn()
     } as unknown as AttachmentHandler
 
-    mockSocialId = 'test-social-id' as PersonId
+    mockToken = 'test-token'
 
     mockWorkspace = {
       getChannel: jest.fn()
     }
 
-    messageManager = new MessageManager(mockCtx, mockClient, mockAttachmentHandler, mockSocialId, mockWorkspace)
+    messageManager = new MessageManager(mockCtx, mockAttachmentHandler, mockToken)
   })
 
   describe('saveMessage', () => {

services/gmail/pod-gmail-next/src/__tests__/parseNameFromEmailHeader.test.ts

@@ -0,0 +1,113 @@
+import { parseNameFromEmailHeader } from '../message/message'
+import { EmailContact } from '../types'
+
+describe('parseNameFromEmailHeader', () => {
+  it('should parse email with name in double quotes', () => {
+    const input = '"John Doe" <john.doe@example.com>'
+    const expected: EmailContact = {
+      email: 'john.doe@example.com',
+      firstName: 'John',
+      lastName: 'Doe'
+    }
+
+    expect(parseNameFromEmailHeader(input)).toEqual(expected)
+  })
+
+  it('should parse email with name without quotes', () => {
+    const input = 'Jane Smith <jane.smith@example.com>'
+    const expected: EmailContact = {
+      email: 'jane.smith@example.com',
+      firstName: 'Jane',
+      lastName: 'Smith'
+    }
+
+    expect(parseNameFromEmailHeader(input)).toEqual(expected)
+  })
+
+  it('should parse email without name', () => {
+    const input = 'no-reply@example.com'
+    const expected: EmailContact = {
+      email: 'no-reply@example.com',
+      firstName: 'no-reply',
+      lastName: 'example.com'
+    }
+
+    expect(parseNameFromEmailHeader(input)).toEqual(expected)
+  })
+
+  it('should parse email with angle brackets only', () => {
+    const input = '<support@example.com>'
+    const expected: EmailContact = {
+      email: 'support@example.com',
+      firstName: 'support',
+      lastName: 'example.com'
+    }
+
+    expect(parseNameFromEmailHeader(input)).toEqual(expected)
+  })
+
+  it('should parse email with multi-word last name', () => {
+    const input = 'Maria Van Der Berg <maria@example.com>'
+    const expected: EmailContact = {
+      email: 'maria@example.com',
+      firstName: 'Maria',
+      lastName: 'Van Der Berg'
+    }
+
+    expect(parseNameFromEmailHeader(input)).toEqual(expected)
+  })
+
+  it('should handle undefined input', () => {
+    const expected: EmailContact = {
+      email: '',
+      firstName: '',
+      lastName: ''
+    }
+
+    expect(parseNameFromEmailHeader(undefined)).toEqual(expected)
+  })
+
+  it('should handle empty string input', () => {
+    const input = ''
+    const expected: EmailContact = {
+      email: '',
+      firstName: '',
+      lastName: ''
+    }
+
+    expect(parseNameFromEmailHeader(input)).toEqual(expected)
+  })
+
+  it('should handle malformed email formats', () => {
+    const input = 'John Doe john.doe@example.com'
+    const expected: EmailContact = {
+      email: 'John Doe john.doe@example.com',
+      firstName: 'John Doe john.doe',
+      lastName: 'example.com'
+    }
+
+    expect(parseNameFromEmailHeader(input)).toEqual(expected)
+  })
+
+  it('should parse single-name format', () => {
+    const input = 'Support <help@example.com>'
+    const expected: EmailContact = {
+      email: 'help@example.com',
+      firstName: 'Support',
+      lastName: 'example.com'
+    }
+
+    expect(parseNameFromEmailHeader(input)).toEqual(expected)
+  })
+
+  it('should handle name with special characters', () => {
+    const input = '"O\'Neill, James" <james.oneill@example.com>'
+    const expected: EmailContact = {
+      email: 'james.oneill@example.com',
+      firstName: "O'Neill,",
+      lastName: 'James'
+    }
+
+    expect(parseNameFromEmailHeader(input)).toEqual(expected)
+  })
+})

services/gmail/pod-gmail-next/src/gmail.ts

@@ -34,7 +34,6 @@ import { MessageManager } from './message/message'
 import { SyncManager } from './message/sync'
 import { getEmail } from './gmail/utils'
 import { Integration } from '@hcengineering/account-client'
-import { GooglePeopleClient } from './gmail/peopleClient'
 
 const SCOPES = ['https://www.googleapis.com/auth/gmail.modify']
 
@@ -105,12 +104,7 @@ export class GmailClient {
     this.client = new TxOperations(client, this.socialId._id)
     this.account = this.user.userId
     this.attachmentHandler = new AttachmentHandler(ctx, workspaceId, storageAdapter, this.gmail, this.client)
-    this.messageManager = new MessageManager(
-      ctx,
-      this.attachmentHandler,
-      this.integrationToken,
-      new GooglePeopleClient(oAuth2Client, ctx, this.rateLimiter)
-    )
+    this.messageManager = new MessageManager(ctx, this.attachmentHandler, this.integrationToken)
     const keyValueClient = getKvsClient(this.integrationToken)
     this.syncManager = new SyncManager(
       ctx,

services/gmail/pod-gmail-next/src/message/message.ts

@@ -12,8 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //
-import { type MeasureContext, Timestamp, AttachedData } from '@hcengineering/core'
-import { type Message } from '@hcengineering/gmail'
+import { type MeasureContext } from '@hcengineering/core'
 import { type GaxiosResponse } from 'gaxios'
 import { gmail_v1 } from 'googleapis'
 import sanitizeHtml from 'sanitize-html'
@@ -21,15 +20,14 @@ import sanitizeHtml from 'sanitize-html'
 import { AttachmentHandler } from './attachments'
 import { decode64 } from '../base64'
 import { createMessages } from './messageCard'
-import { GooglePeopleClient } from '../gmail/peopleClient'
 import { randomUUID } from 'crypto'
+import { ConvertedMessage, EmailContact } from '../types'
 
 export class MessageManager {
   constructor (
     private readonly ctx: MeasureContext,
     private readonly attachmentHandler: AttachmentHandler,
-    private readonly token: string,
-    private readonly peopleClient: GooglePeopleClient
+    private readonly token: string
   ) {}
 
   async saveMessage (message: GaxiosResponse<gmail_v1.Schema$Message>, me: string): Promise<void> {
@@ -38,14 +36,14 @@ export class MessageManager {
 
     await createMessages(
       this.ctx,
-      this.peopleClient,
       this.token,
-      message.data.id ?? randomUUID(),
+      res.messageId ?? randomUUID(),
       res.from,
       [res.to, ...(res.copy ?? [])],
       res.subject,
       res.content,
       attachments,
+      me,
       res.replyTo
     )
   }
@@ -88,18 +86,16 @@ function getPartMessage (part: gmail_v1.Schema$MessagePart | undefined, mime: st
   return getPartsMessage(part.parts, mime)
 }
 
-function convertMessage (
-  message: GaxiosResponse<gmail_v1.Schema$Message>,
-  me: string
-): AttachedData<Message> & { modifiedOn: Timestamp } {
+function convertMessage (message: GaxiosResponse<gmail_v1.Schema$Message>, me: string): ConvertedMessage {
   const date = message.data.internalDate != null ? new Date(Number.parseInt(message.data.internalDate)) : new Date()
-  const from = getHeaderValue(message.data.payload, 'From') ?? ''
-  const to = getHeaderValue(message.data.payload, 'To') ?? ''
+  const from = parseNameFromEmailHeader(getHeaderValue(message.data.payload, 'From') ?? '')
+  const to = parseNameFromEmailHeader(getHeaderValue(message.data.payload, 'To') ?? '')
+
   const copy =
     getHeaderValue(message.data.payload, 'Cc')
       ?.split(',')
-      .map((p) => p.trim()) ?? undefined
-  const incoming = !from.includes(me)
+      .map((p) => parseNameFromEmailHeader(p.trim())) ?? undefined
+  const incoming = !from.email.includes(me)
   return {
     modifiedOn: date.getTime(),
     messageId: getHeaderValue(message.data.payload, 'Message-ID') ?? '',
@@ -114,3 +110,57 @@ function convertMessage (
     sendOn: date.getTime()
   }
 }
+
+export function parseNameFromEmailHeader (headerValue: string | undefined): EmailContact {
+  if (headerValue == null || headerValue.trim() === '') {
+    return {
+      email: '',
+      firstName: '',
+      lastName: ''
+    }
+  }
+
+  // Match pattern like: "Name" <email@example.com> or Name <email@example.com>
+  const nameEmailPattern = /^(?:"?([^"<]+)"?\s*)?<([^>]+)>$/
+  const match = headerValue.trim().match(nameEmailPattern)
+
+  if (match == null) {
+    const address = headerValue.trim()
+    const parts = address.split('@')
+    return {
+      email: address,
+      firstName: parts[0],
+      lastName: parts[1]
+    }
+  }
+
+  const displayName = match[1]?.trim()
+  const email = match[2].trim()
+
+  if (displayName == null || displayName === '') {
+    const parts = email.split('@')
+    return {
+      email,
+      firstName: parts[0],
+      lastName: parts[1]
+    }
+  }
+
+  const nameParts = displayName.split(/\s+/)
+  let firstName: string | undefined
+  let lastName: string | undefined
+
+  if (nameParts.length === 1) {
+    firstName = nameParts[0]
+  } else if (nameParts.length > 1) {
+    firstName = nameParts[0]
+    lastName = nameParts.slice(1).join(' ')
+  }
+
+  const parts = email.split('@')
+  return {
+    email,
+    firstName: firstName ?? parts[0],
+    lastName: lastName ?? parts[1]
+  }
+}

services/gmail/pod-gmail-next/src/message/messageCard.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 import { getClient as getAccountClient, isWorkspaceLoginInfo } from '@hcengineering/account-client'
-import { createRestTxOperations } from '@hcengineering/api-client'
+import { createRestTxOperations, createRestClient } from '@hcengineering/api-client'
 import { type Card } from '@hcengineering/card'
 import {
   type RestClient as CommunicationClient,
@@ -27,28 +27,31 @@ import {
   type PersonId,
   type Ref,
   type TxOperations,
+  type Doc,
   generateId,
   PersonUuid,
-  RateLimiter
+  RateLimiter,
+  SocialIdType
 } from '@hcengineering/core'
 import mail from '@hcengineering/mail'
+import chat from '@hcengineering/chat'
+
 import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
 
 import config from '../config'
-import { ensureGlobalPerson, ensureLocalPerson } from './person'
 import { type AttachedFile } from './types'
-import { GooglePeopleClient } from '../gmail/peopleClient'
+import { EmailContact } from '../types'
 
 export async function createMessages (
   ctx: MeasureContext,
-  peopleClient: GooglePeopleClient,
   token: string,
   mailId: string,
-  from: string,
-  tos: string[],
+  from: EmailContact,
+  tos: EmailContact[],
   subject: string,
   content: string,
   attachments: AttachedFile[],
+  me: string,
   inReplyTo?: string
 ): Promise<void> {
   ctx.info('Sending message', { mailId, from, to: tos.join(',') })
@@ -64,54 +67,20 @@ export async function createMessages (
   const transactorUrl = wsInfo.endpoint.replace('ws://', 'http://').replace('wss://', 'https://')
   const txClient = await createRestTxOperations(transactorUrl, wsInfo.workspace, wsInfo.token)
   const msgClient = getCommunicationClient(wsInfo.endpoint, wsInfo.workspace, wsInfo.token)
+  const restClient = createRestClient(transactorUrl, wsInfo.workspace, wsInfo.token)
 
-  const fromPerson = await ensureGlobalPerson(ctx, accountClient, mailId, from, peopleClient)
-  if (fromPerson === undefined) {
-    ctx.error('Unable to create message without a proper FROM', { mailId })
-    return
-  }
-  try {
-    await ensureLocalPerson(
-      ctx,
-      txClient,
-      mailId,
-      fromPerson.uuid,
-      fromPerson.socialId,
-      from,
-      fromPerson.firstName,
-      fromPerson.lastName
-    )
-  } catch (error) {
-    ctx.error('Failed to ensure local FROM person', { error, mailId })
-    ctx.error('Unable to create message without a proper FROM', { mailId })
-    return
-  }
+  const fromPerson = await restClient.ensurePerson(SocialIdType.EMAIL, from.email, from.firstName, from.lastName)
 
   const toPersons: { address: string, uuid: PersonUuid, socialId: PersonId }[] = []
   for (const to of tos) {
-    const toPerson = await ensureGlobalPerson(ctx, accountClient, mailId, to, peopleClient)
+    const toPerson = await restClient.ensurePerson(SocialIdType.EMAIL, to.email, to.firstName, to.lastName)
     if (toPerson === undefined) {
       continue
     }
-    try {
-      await ensureLocalPerson(
-        ctx,
-        txClient,
-        mailId,
-        toPerson.uuid,
-        toPerson.socialId,
-        to,
-        toPerson.firstName,
-        toPerson.lastName
-      )
-    } catch (error) {
-      ctx.error('Failed to ensure local TO person, skip', { error, mailId })
-      continue
-    }
-    toPersons.push({ address: to, ...toPerson })
+    toPersons.push({ address: to.email, ...toPerson })
   }
   if (toPersons.length === 0) {
-    ctx.error('Unable to create message without a proper TO', { mailId })
+    ctx.error('Unable to create message without a proper TO', { mailId, from })
     return
   }
 
@@ -148,7 +117,7 @@ export async function createMessages (
   }
 
   try {
-    const spaces = await getPersonSpaces(ctx, txClient, mailId, fromPerson.uuid, from)
+    const spaces = await getPersonSpaces(ctx, txClient, mailId, fromPerson.uuid, from.email)
     if (spaces.length > 0) {
       await saveMessageToSpaces(
         ctx,
@@ -161,6 +130,7 @@ export async function createMessages (
         subject,
         content,
         attachedBlobs,
+        me,
         inReplyTo
       )
     }
@@ -188,6 +158,7 @@ export async function createMessages (
           subject,
           content,
           attachedBlobs,
+          me,
           inReplyTo
         )
       }
@@ -208,7 +179,7 @@ async function getPersonSpaces (
   const personRefs = persons.map((p) => p._id)
   const spaces = await client.findAll(contact.class.PersonSpace, { person: { $in: personRefs } })
   if (spaces.length === 0) {
-    ctx.info('No personal space found, skip', { mailId, personUuid, email })
+    ctx.warn('No personal space found, skip', { mailId, personUuid, email })
   }
   return spaces
 }
@@ -224,6 +195,7 @@ async function saveMessageToSpaces (
   subject: string,
   content: string,
   attachments: AttachedFile[],
+  me: string,
   inReplyTo?: string
 ): Promise<void> {
   const rateLimiter = new RateLimiter(10)
@@ -247,8 +219,9 @@ async function saveMessageToSpaces (
         }
       }
       if (threadId === undefined) {
+        const channel = await getOrCreateChannel(ctx, client, spaceId, participants, modifiedBy, me)
         const newThreadId = await client.createDoc(
-          mail.class.MailThread,
+          chat.masterTag.Thread,
           space._id,
           {
             title: subject,
@@ -257,7 +230,8 @@ async function saveMessageToSpaces (
             members: participants,
             archived: false,
             createdBy: modifiedBy,
-            modifiedBy
+            modifiedBy,
+            parent: channel
           },
           generateId(),
           undefined,
@@ -269,12 +243,12 @@ async function saveMessageToSpaces (
 
       const { id: messageId, created: messageCreated } = await msgClient.createMessage(
         threadId,
-        mail.class.MailThread,
+        chat.masterTag.Thread,
         content,
         modifiedBy,
         MessageType.Message
       )
-      ctx.info('Created message', { mailId, messageId, threadId })
+      ctx.info('Created message', { mailId, messageId, threadId, content })
 
       for (const a of attachments) {
         await msgClient.createFile(
@@ -304,3 +278,37 @@ async function saveMessageToSpaces (
   }
   await rateLimiter.waitProcessing()
 }
+
+async function getOrCreateChannel (
+  ctx: MeasureContext,
+  client: TxOperations,
+  space: Ref<PersonSpace>,
+  participants: PersonId[],
+  modifiedBy: PersonId,
+  me: string
+): Promise<Ref<Doc> | undefined> {
+  try {
+    const channel = await client.findOne(chat.masterTag.Channel, { title: me })
+    ctx.info('Existing channel', { me, space, channel })
+    if (channel != null) return channel._id
+    ctx.info('Creating new channel', { me, space })
+    return await client.createDoc(
+      chat.masterTag.Channel,
+      space,
+      {
+        title: me,
+        private: true,
+        members: participants,
+        archived: false,
+        createdBy: modifiedBy,
+        modifiedBy
+      },
+      generateId(),
+      undefined,
+      modifiedBy
+    )
+  } catch (err: any) {
+    ctx.error('Failed to create channel', { me, space })
+    return undefined
+  }
+}

services/gmail/pod-gmail-next/src/message/person.ts

@@ -24,30 +24,31 @@ import {
 import contact, { AvatarType, combineName, SocialIdentityRef } from '@hcengineering/contact'
 import { AccountClient } from '@hcengineering/account-client'
 
-import { GooglePeopleClient } from '../gmail/peopleClient'
+import { EmailContact } from '../types'
 
 export async function ensureGlobalPerson (
   ctx: MeasureContext,
   client: AccountClient,
   mailId: string,
-  email: string,
-  peopleClient: GooglePeopleClient
+  contact: EmailContact
 ): Promise<{ socialId: PersonId, uuid: PersonUuid, firstName: string, lastName: string } | undefined> {
-  const googlePerson = await peopleClient.getContactInfo(email)
-  const firstName = googlePerson?.firstName ?? email
-  const lastName = googlePerson?.lastName ?? ''
-  const socialKey = buildSocialIdString({ type: SocialIdType.EMAIL, value: email })
+  const socialKey = buildSocialIdString({ type: SocialIdType.EMAIL, value: contact.email })
   const socialId = await client.findSocialIdBySocialKey(socialKey)
   const uuid = await client.findPersonBySocialKey(socialKey)
   if (socialId !== undefined && uuid !== undefined) {
-    return { socialId, uuid, firstName, lastName }
+    return { socialId, uuid, firstName: contact.firstName, lastName: contact.lastName }
   }
   try {
-    const globalPerson = await client.ensurePerson(SocialIdType.EMAIL, email, firstName, lastName)
-    ctx.info('Created global person', { mailId, email, personUuid: globalPerson.uuid })
-    return { ...globalPerson, firstName, lastName }
+    const globalPerson = await client.ensurePerson(
+      SocialIdType.EMAIL,
+      contact.email,
+      contact.firstName,
+      contact.lastName
+    )
+    ctx.info('Created global person', { mailId, email: contact, personUuid: globalPerson.uuid })
+    return { ...globalPerson, firstName: contact.firstName, lastName: contact.lastName }
   } catch (error) {
-    ctx.error('Failed to create global person', { mailId, error, email })
+    ctx.error('Failed to create global person', { mailId, error, email: contact })
   }
   return undefined
 }

services/gmail/pod-gmail-next/src/types.ts

@@ -38,11 +38,32 @@ export type State = User & {
 }
 
 export interface AttachedFile {
-  size?: number
-  file: string
-  type?: string
-  lastModified: number
+  id: string
   name: string
+  data: Buffer
+  contentType: string
+  size?: number
+  lastModified: number
+}
+
+export interface EmailContact {
+  email: string
+  firstName: string
+  lastName: string
+}
+
+export interface ConvertedMessage {
+  modifiedOn: number
+  messageId: string
+  replyTo?: string
+  copy?: EmailContact[]
+  content: string
+  textContent: string
+  from: EmailContact
+  to: EmailContact
+  incoming: boolean
+  subject: string
+  sendOn: number
 }
 
 export type Channel = Pick<PlatformChannel, 'value' | keyof Doc>

services/gmail/pod-gmail/src/gmail.ts

@@ -371,7 +371,7 @@ export class GmailClient {
     try {
       this.ctx.info('Register client', { socialId: this.socialId._id, email: this.email })
       const controller = GmailController.getGmailController()
-      controller.addClient(this.socialId._id, this.user.workspace, this)
+      controller.addClient(this.socialId._id, this)
     } catch (err) {
       this.ctx.error('Add client error', {
         workspaceUuid: this.user.workspace,