mirror of
https://github.com/hcengineering/platform.git
synced 2025-06-09 09:20:54 +00:00
UBERF-11004: Fix mta content parsing
Signed-off-by: Artem Savchenko <armisav@gmail.com>
This commit is contained in:
Artem Savchenko
parent
18f00908f1
commit
6ac0c09c7a
@ -430,13 +430,12 @@ describe('handleMtaHook', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should process multipart email with both HTML and text correctly', async () => {
|
||||
it('should process email plain/text content header', async () => {
|
||||
// Create a multipart email with both text and HTML
|
||||
const textContent = 'This is the plain text version'
|
||||
const htmlContent = '<html><body><p>This is the HTML version</p></body></html>'
|
||||
|
||||
// Mock message with multipart content by setting multiple headers and contents
|
||||
const multipartMessage = {
|
||||
const message = {
|
||||
envelope: {
|
||||
from: { address: 'sender@example.com' },
|
||||
to: [{ address: 'recipient@example.com' }]
|
||||
@ -444,26 +443,17 @@ describe('handleMtaHook', () => {
|
||||
message: {
|
||||
headers: [
|
||||
['Content-Type', 'multipart/alternative; boundary="boundary-string"'],
|
||||
['Subject', 'Multipart Test Email'],
|
||||
['Subject', 'Test Email'],
|
||||
['From', 'Sender <sender@example.com>'],
|
||||
['To', 'Recipient <recipient@example.com>']
|
||||
],
|
||||
contents: [
|
||||
{
|
||||
headers: [['Content-Type', 'text/plain; charset=utf-8']],
|
||||
content: textContent
|
||||
},
|
||||
{
|
||||
headers: [['Content-Type', 'text/html; charset=utf-8']],
|
||||
content: htmlContent
|
||||
}
|
||||
]
|
||||
contents: `Content-Type: text/plain; charset=utf-8 \r\n${textContent}`
|
||||
}
|
||||
}
|
||||
|
||||
mockReq = {
|
||||
headers: { 'x-hook-token': 'test-hook-token' },
|
||||
body: multipartMessage
|
||||
body: message
|
||||
}
|
||||
|
||||
await handleMtaHook(mockReq as Request, mockRes as Response, mockCtx)
|
||||
@ -483,82 +473,13 @@ describe('handleMtaHook', () => {
|
||||
mockLoginInfo,
|
||||
expect.objectContaining({
|
||||
mailId: expect.any(String),
|
||||
from: { email: 'sender@example.com', firstName: 'Sender', lastName: '' },
|
||||
to: [{ email: 'recipient@example.com', firstName: 'Recipient', lastName: '' }],
|
||||
subject: 'Multipart Test Email',
|
||||
content: 'This is the HTML version',
|
||||
from: { email: 'sender@example.com', firstName: 'Sender', lastName: 'example.com' },
|
||||
to: [{ email: 'recipient@example.com', firstName: 'Recipient', lastName: 'example.com' }],
|
||||
subject: 'Test Email',
|
||||
content: textContent,
|
||||
incoming: true
|
||||
}),
|
||||
[]
|
||||
)
|
||||
})
|
||||
|
||||
it('should handle HTML email with inline images correctly', async () => {
|
||||
// HTML content with embedded image reference
|
||||
const htmlWithImage =
|
||||
'<html><body><p>Test with image:</p><img src="cid:image1@example.com" alt="Test Image"></body></html>'
|
||||
|
||||
// Create image attachment
|
||||
const imageAttachment = {
|
||||
headers: [
|
||||
['Content-Type', 'image/jpeg'],
|
||||
['Content-Disposition', 'inline; filename="image.jpg"'],
|
||||
['Content-ID', '<image1@example.com>']
|
||||
],
|
||||
content: 'base64encodedcontent' // Would normally be a Base64 string
|
||||
}
|
||||
|
||||
// Create multipart message with HTML and image
|
||||
const multipartMessage = {
|
||||
envelope: {
|
||||
from: { address: 'sender@example.com' },
|
||||
to: [{ address: 'recipient@example.com' }]
|
||||
},
|
||||
message: {
|
||||
headers: [
|
||||
['Content-Type', 'multipart/related; boundary="boundary-string"'],
|
||||
['Subject', 'Email with Inline Image'],
|
||||
['From', 'Sender <sender@example.com>'],
|
||||
['To', 'Recipient <recipient@example.com>']
|
||||
],
|
||||
contents: [
|
||||
{
|
||||
headers: [['Content-Type', 'text/html; charset=utf-8']],
|
||||
content: htmlWithImage
|
||||
},
|
||||
imageAttachment
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
mockReq = {
|
||||
headers: { 'x-hook-token': 'test-hook-token' },
|
||||
body: multipartMessage
|
||||
}
|
||||
|
||||
await handleMtaHook(mockReq as Request, mockRes as Response, mockCtx)
|
||||
|
||||
// Should process message with attachments
|
||||
expect(createMessages).toHaveBeenCalledWith(
|
||||
client.baseConfig,
|
||||
mockCtx,
|
||||
mockTxOperations,
|
||||
{},
|
||||
{},
|
||||
client.mailServiceToken,
|
||||
mockLoginInfo,
|
||||
expect.objectContaining({
|
||||
htmlContent: htmlWithImage
|
||||
// Other fields as expected
|
||||
}),
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
contentType: 'image/jpeg',
|
||||
name: 'image.jpg',
|
||||
contentId: '<image1@example.com>'
|
||||
// Other attachment fields
|
||||
})
|
||||
])
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
311
services/mail/pod-inbound-mail/src/__tests__/utils.test.ts
Normal file
311
services/mail/pod-inbound-mail/src/__tests__/utils.test.ts
Normal file
@ -0,0 +1,311 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { parseContent, getHeader, removeContentTypeHeader } from '../utils'
|
||||
import { MeasureContext } from '@hcengineering/core'
|
||||
import { MtaMessage } from '../types'
|
||||
import { readEml } from 'eml-parse-js'
|
||||
|
||||
// Mock dependencies
|
||||
jest.mock('eml-parse-js')
|
||||
jest.mock('sanitize-html', () => (html: string) => html)
|
||||
jest.mock('turndown', () => {
|
||||
return class TurndownService {
|
||||
turndown (html: string): string {
|
||||
return html.replace(/<\/?[^>]+(>|$)/g, '') // Simple HTML tag removal
|
||||
}
|
||||
}
|
||||
})
|
||||
jest.mock('../config', () => ({
|
||||
__esModule: true,
|
||||
default: {
|
||||
storageConfig: {}
|
||||
}
|
||||
}))
|
||||
|
||||
describe('utils.ts', () => {
|
||||
let mockCtx: MeasureContext
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks()
|
||||
mockCtx = {
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
info: jest.fn()
|
||||
} as unknown as MeasureContext
|
||||
|
||||
// Default mock implementation for readEml
|
||||
;(readEml as jest.Mock).mockImplementation((content, callback) => {
|
||||
callback(null, {
|
||||
text: 'Plain text content',
|
||||
html: '<p>HTML content</p>',
|
||||
attachments: []
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseContent', () => {
|
||||
it('should handle plain text emails', async () => {
|
||||
// Arrange
|
||||
const plainTextMessage: MtaMessage = {
|
||||
envelope: {
|
||||
from: { address: 'sender@example.com' },
|
||||
to: [{ address: 'recipient@example.com' }]
|
||||
},
|
||||
message: {
|
||||
headers: [['Content-Type', 'text/plain; charset=utf-8']],
|
||||
contents: 'This is a plain text email'
|
||||
}
|
||||
}
|
||||
|
||||
// Act
|
||||
const result = await parseContent(mockCtx, plainTextMessage)
|
||||
|
||||
// Assert
|
||||
expect(result).toEqual({
|
||||
content: 'This is a plain text email',
|
||||
attachments: []
|
||||
})
|
||||
expect(readEml).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should handle HTML emails', async () => {
|
||||
// Arrange
|
||||
const htmlMessage: MtaMessage = {
|
||||
envelope: {
|
||||
from: { address: 'sender@example.com' },
|
||||
to: [{ address: 'recipient@example.com' }]
|
||||
},
|
||||
message: {
|
||||
headers: [['Content-Type', 'text/html; charset=utf-8']],
|
||||
contents: '<html><body><p>This is an HTML email</p></body></html>'
|
||||
}
|
||||
} as any
|
||||
|
||||
// Mock readEml to return HTML content
|
||||
;(readEml as jest.Mock).mockImplementation((content, callback) => {
|
||||
callback(null, {
|
||||
text: '',
|
||||
html: '<p>This is an HTML email</p>',
|
||||
attachments: []
|
||||
})
|
||||
})
|
||||
|
||||
// Act
|
||||
const result = await parseContent(mockCtx, htmlMessage)
|
||||
|
||||
// Assert
|
||||
expect(result).toEqual({
|
||||
content: 'This is an HTML email',
|
||||
attachments: []
|
||||
})
|
||||
})
|
||||
|
||||
it('should handle multipart emails with text and HTML parts', async () => {
|
||||
// Arrange
|
||||
const multipartMessage: MtaMessage = {
|
||||
envelope: {
|
||||
from: { address: 'sender@example.com' },
|
||||
to: [{ address: 'recipient@example.com' }]
|
||||
},
|
||||
message: {
|
||||
headers: [['Content-Type', 'multipart/alternative; boundary="boundary"']],
|
||||
contents:
|
||||
'--boundary\r\nContent-Type: text/plain\r\n\r\nText part\r\n--boundary\r\nContent-Type: text/html\r\n\r\n<p>HTML part</p>\r\n--boundary--'
|
||||
}
|
||||
} as any
|
||||
|
||||
// Mock readEml to return both text and HTML content
|
||||
;(readEml as jest.Mock).mockImplementation((content, callback) => {
|
||||
callback(null, {
|
||||
text: 'Text part',
|
||||
html: '<p>HTML part</p>',
|
||||
attachments: []
|
||||
})
|
||||
})
|
||||
|
||||
// Act
|
||||
const result = await parseContent(mockCtx, multipartMessage)
|
||||
|
||||
// Assert
|
||||
expect(result).toEqual({
|
||||
content: 'HTML part',
|
||||
attachments: []
|
||||
})
|
||||
})
|
||||
|
||||
it('should throw error when Content-Type header is not found', async () => {
|
||||
// Arrange
|
||||
const messageWithNoContentType: MtaMessage = {
|
||||
envelope: {
|
||||
from: { address: 'sender@example.com' },
|
||||
to: [{ address: 'recipient@example.com' }]
|
||||
},
|
||||
message: {
|
||||
headers: [['Subject', 'Test Email']],
|
||||
contents: 'Email content'
|
||||
}
|
||||
}
|
||||
|
||||
// Act & Assert
|
||||
await expect(parseContent(mockCtx, messageWithNoContentType)).rejects.toThrow('Content-Type header not found')
|
||||
})
|
||||
})
|
||||
|
||||
describe('getHeader', () => {
|
||||
it('should return the value of the specified header', () => {
|
||||
// Arrange
|
||||
const message: MtaMessage = {
|
||||
envelope: {
|
||||
from: { address: 'sender@example.com' },
|
||||
to: [{ address: 'recipient@example.com' }]
|
||||
},
|
||||
message: {
|
||||
headers: [
|
||||
['Subject', 'Test Email'],
|
||||
['Content-Type', 'text/plain'],
|
||||
['X-Custom-Header', 'Custom Value']
|
||||
],
|
||||
contents: 'Email content'
|
||||
}
|
||||
}
|
||||
|
||||
// Act & Assert
|
||||
expect(getHeader(message, 'Subject')).toBe('Test Email')
|
||||
expect(getHeader(message, 'Content-Type')).toBe('text/plain')
|
||||
expect(getHeader(message, 'X-Custom-Header')).toBe('Custom Value')
|
||||
})
|
||||
|
||||
it('should be case-insensitive when looking for headers', () => {
|
||||
// Arrange
|
||||
const message: MtaMessage = {
|
||||
envelope: {
|
||||
from: { address: 'sender@example.com' },
|
||||
to: [{ address: 'recipient@example.com' }]
|
||||
},
|
||||
message: {
|
||||
headers: [
|
||||
['Subject', 'Test Email'],
|
||||
['Content-Type', 'text/plain']
|
||||
],
|
||||
contents: 'Email content'
|
||||
}
|
||||
}
|
||||
|
||||
// Act & Assert
|
||||
expect(getHeader(message, 'subject')).toBe('Test Email')
|
||||
expect(getHeader(message, 'CONTENT-TYPE')).toBe('text/plain')
|
||||
})
|
||||
|
||||
it('should return undefined for non-existent headers', () => {
|
||||
// Arrange
|
||||
const message: MtaMessage = {
|
||||
envelope: {
|
||||
from: { address: 'sender@example.com' },
|
||||
to: [{ address: 'recipient@example.com' }]
|
||||
},
|
||||
message: {
|
||||
headers: [['Subject', 'Test Email']],
|
||||
contents: 'Email content'
|
||||
}
|
||||
}
|
||||
|
||||
// Act & Assert
|
||||
expect(getHeader(message, 'X-Not-Exists')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should trim the header value', () => {
|
||||
// Arrange
|
||||
const message: MtaMessage = {
|
||||
envelope: {
|
||||
from: { address: 'sender@example.com' },
|
||||
to: [{ address: 'recipient@example.com' }]
|
||||
},
|
||||
message: {
|
||||
headers: [['Subject', ' Test Email ']],
|
||||
contents: 'Email content'
|
||||
}
|
||||
}
|
||||
|
||||
// Act & Assert
|
||||
expect(getHeader(message, 'Subject')).toBe('Test Email')
|
||||
})
|
||||
})
|
||||
|
||||
describe('removeContentTypeHeader', () => {
|
||||
it('should remove Content-Type header from content', () => {
|
||||
// Arrange
|
||||
const content = 'Content-Type: text/plain; charset=utf-8\r\nHello world'
|
||||
|
||||
// Act
|
||||
const result = removeContentTypeHeader(content)
|
||||
|
||||
// Assert
|
||||
expect(result).toBe('Hello world')
|
||||
})
|
||||
|
||||
it('should handle content with no Content-Type header', () => {
|
||||
// Arrange
|
||||
const content = 'Hello world'
|
||||
|
||||
// Act
|
||||
const result = removeContentTypeHeader(content)
|
||||
|
||||
// Assert
|
||||
expect(result).toBe('Hello world')
|
||||
})
|
||||
|
||||
it('should handle content with Content-Type header in different case', () => {
|
||||
// Arrange
|
||||
const content = 'content-type: text/plain; charset=utf-8\r\nHello world'
|
||||
|
||||
// Act
|
||||
const result = removeContentTypeHeader(content)
|
||||
|
||||
// Assert
|
||||
expect(result).toBe('Hello world')
|
||||
})
|
||||
|
||||
it('should handle content with multiple headers', () => {
|
||||
// Arrange
|
||||
const content =
|
||||
'Subject: Test Email\r\nContent-Type: text/plain; charset=utf-8\r\nFrom: test@example.com\r\n\r\nHello world'
|
||||
|
||||
// Act
|
||||
const result = removeContentTypeHeader(content)
|
||||
|
||||
// Assert
|
||||
expect(result).toBe('Subject: Test Email\r\nFrom: test@example.com\r\n\r\nHello world')
|
||||
})
|
||||
|
||||
it('should handle null or undefined content', () => {
|
||||
// Act & Assert
|
||||
expect(removeContentTypeHeader(null as any)).toBeNull()
|
||||
expect(removeContentTypeHeader(undefined as any)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should handle different line endings', () => {
|
||||
// Arrange
|
||||
const crlfContent = 'Content-Type: text/plain\r\nHello world'
|
||||
const lfContent = 'Content-Type: text/plain\nHello world'
|
||||
const crContent = 'Content-Type: text/plain\rHello world'
|
||||
|
||||
// Act & Assert
|
||||
expect(removeContentTypeHeader(crlfContent)).toBe('Hello world')
|
||||
expect(removeContentTypeHeader(lfContent)).toBe('Hello world')
|
||||
expect(removeContentTypeHeader(crContent)).toBe('Hello world')
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -15,12 +15,7 @@
|
||||
import { createHash } from 'crypto'
|
||||
import { Request, Response } from 'express'
|
||||
import { MeasureContext } from '@hcengineering/core'
|
||||
import {
|
||||
type EmailContact,
|
||||
type EmailMessage,
|
||||
createMessages,
|
||||
getProducer
|
||||
} from '@hcengineering/mail-common'
|
||||
import { type EmailContact, type EmailMessage, createMessages, getProducer } from '@hcengineering/mail-common'
|
||||
import { getClient as getAccountClient } from '@hcengineering/account-client'
|
||||
import { createRestTxOperations } from '@hcengineering/api-client'
|
||||
|
||||
|
||||
@ -17,9 +17,7 @@ import { readEml, ReadedEmlJson } from 'eml-parse-js'
|
||||
import TurndownService from 'turndown'
|
||||
import sanitizeHtml from 'sanitize-html'
|
||||
import { MeasureContext } from '@hcengineering/core'
|
||||
import {
|
||||
type Attachment
|
||||
} from '@hcengineering/mail-common'
|
||||
import { type Attachment } from '@hcengineering/mail-common'
|
||||
|
||||
import { MtaMessage } from './types'
|
||||
import config from './config'
|
||||
@ -37,10 +35,11 @@ export async function parseContent (
|
||||
return { content: mta.message.contents, attachments: [] }
|
||||
}
|
||||
|
||||
// TODO: UBERF-11029 - remove this logging after testing
|
||||
console.log('Parsing email content', mta.message.contents)
|
||||
const email = await getEmailContent(mta.message.contents)
|
||||
|
||||
let content = email.text ?? ''
|
||||
console.log('Content:', content)
|
||||
let isMarkdown = false
|
||||
if (email.html !== undefined) {
|
||||
try {
|
||||
@ -116,12 +115,21 @@ async function getEmailContent (mtaContent: string): Promise<ReadedEmlJson> {
|
||||
if (isEmptyString(email.text) && isEmptyString(email.html)) {
|
||||
return {
|
||||
...email,
|
||||
text: mtaContent
|
||||
text: removeContentTypeHeader(mtaContent)
|
||||
}
|
||||
}
|
||||
return email
|
||||
}
|
||||
|
||||
export function removeContentTypeHeader (content: string): string {
|
||||
if (content == null) {
|
||||
return content
|
||||
}
|
||||
|
||||
const contentTypeRegex = /^Content-Type:.*?(?:\r\n|\n|\r)/im
|
||||
return content.replace(contentTypeRegex, '')
|
||||
}
|
||||
|
||||
function guessContentType (content: string): string {
|
||||
// Simple heuristic - if it contains HTML tags, it's likely HTML
|
||||
if (/<[a-z][\s\S]*>/i.test(content)) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user