mirror of
https://github.com/hcengineering/platform.git
synced 2025-06-12 13:42:38 +00:00
UBERF-11004: Fix mta-hook email content
Signed-off-by: Artem Savchenko <armisav@gmail.com>
This commit is contained in:
Artem Savchenko
parent
02f584811f
commit
18f00908f1
@ -16,10 +16,12 @@
|
|||||||
import { Request, Response } from 'express'
|
import { Request, Response } from 'express'
|
||||||
import { MeasureContext } from '@hcengineering/core'
|
import { MeasureContext } from '@hcengineering/core'
|
||||||
import { createMessages } from '@hcengineering/mail-common'
|
import { createMessages } from '@hcengineering/mail-common'
|
||||||
import { type MtaMessage, handleMtaHook } from '../handlerMta'
|
|
||||||
import * as client from '../client'
|
|
||||||
import { createRestTxOperations } from '@hcengineering/api-client'
|
import { createRestTxOperations } from '@hcengineering/api-client'
|
||||||
|
|
||||||
|
import { handleMtaHook } from '../handlerMta'
|
||||||
|
import * as client from '../client'
|
||||||
|
import { type MtaMessage } from '../types'
|
||||||
|
|
||||||
// Mock dependencies
|
// Mock dependencies
|
||||||
jest.mock('@hcengineering/mail-common', () => ({
|
jest.mock('@hcengineering/mail-common', () => ({
|
||||||
createMessages: jest.fn(),
|
createMessages: jest.fn(),
|
||||||
@ -388,4 +390,175 @@ describe('handleMtaHook', () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
it('should process HTML email correctly', async () => {
|
||||||
|
// Mock request with HTML content
|
||||||
|
const htmlContent = '<html><body><h1>Hello</h1><p>This is an <b>HTML</b> test email</p></body></html>'
|
||||||
|
mockReq = {
|
||||||
|
headers: { 'x-hook-token': 'test-hook-token' },
|
||||||
|
body: createValidMtaMessage('sender@example.com', ['recipient@example.com'], {
|
||||||
|
subject: 'HTML Test Subject',
|
||||||
|
contentType: 'text/html; charset=utf-8',
|
||||||
|
content: htmlContent
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
await handleMtaHook(mockReq as Request, mockRes as Response, mockCtx)
|
||||||
|
|
||||||
|
// Should return 200
|
||||||
|
expect(mockStatus).toHaveBeenCalledWith(200)
|
||||||
|
expect(mockSend).toHaveBeenCalledWith({ action: 'accept' })
|
||||||
|
|
||||||
|
// Should process the message with both HTML and text content
|
||||||
|
expect(createMessages).toHaveBeenCalledWith(
|
||||||
|
client.baseConfig,
|
||||||
|
mockCtx,
|
||||||
|
mockTxOperations,
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
client.mailServiceToken,
|
||||||
|
mockLoginInfo,
|
||||||
|
expect.objectContaining({
|
||||||
|
mailId: expect.any(String),
|
||||||
|
from: { email: 'sender@example.com', firstName: 'sender', lastName: 'example.com' },
|
||||||
|
to: [{ email: 'recipient@example.com', firstName: 'recipient', lastName: 'example.com' }],
|
||||||
|
subject: 'HTML Test Subject',
|
||||||
|
content: htmlContent,
|
||||||
|
incoming: true
|
||||||
|
}),
|
||||||
|
[] // attachments
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should process multipart email with both HTML and text correctly', async () => {
|
||||||
|
// Create a multipart email with both text and HTML
|
||||||
|
const textContent = 'This is the plain text version'
|
||||||
|
const htmlContent = '<html><body><p>This is the HTML version</p></body></html>'
|
||||||
|
|
||||||
|
// Mock message with multipart content by setting multiple headers and contents
|
||||||
|
const multipartMessage = {
|
||||||
|
envelope: {
|
||||||
|
from: { address: 'sender@example.com' },
|
||||||
|
to: [{ address: 'recipient@example.com' }]
|
||||||
|
},
|
||||||
|
message: {
|
||||||
|
headers: [
|
||||||
|
['Content-Type', 'multipart/alternative; boundary="boundary-string"'],
|
||||||
|
['Subject', 'Multipart Test Email'],
|
||||||
|
['From', 'Sender <sender@example.com>'],
|
||||||
|
['To', 'Recipient <recipient@example.com>']
|
||||||
|
],
|
||||||
|
contents: [
|
||||||
|
{
|
||||||
|
headers: [['Content-Type', 'text/plain; charset=utf-8']],
|
||||||
|
content: textContent
|
||||||
|
},
|
||||||
|
{
|
||||||
|
headers: [['Content-Type', 'text/html; charset=utf-8']],
|
||||||
|
content: htmlContent
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
mockReq = {
|
||||||
|
headers: { 'x-hook-token': 'test-hook-token' },
|
||||||
|
body: multipartMessage
|
||||||
|
}
|
||||||
|
|
||||||
|
await handleMtaHook(mockReq as Request, mockRes as Response, mockCtx)
|
||||||
|
|
||||||
|
// Should return 200
|
||||||
|
expect(mockStatus).toHaveBeenCalledWith(200)
|
||||||
|
expect(mockSend).toHaveBeenCalledWith({ action: 'accept' })
|
||||||
|
|
||||||
|
// Should process the message with both content types
|
||||||
|
expect(createMessages).toHaveBeenCalledWith(
|
||||||
|
client.baseConfig,
|
||||||
|
mockCtx,
|
||||||
|
mockTxOperations,
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
client.mailServiceToken,
|
||||||
|
mockLoginInfo,
|
||||||
|
expect.objectContaining({
|
||||||
|
mailId: expect.any(String),
|
||||||
|
from: { email: 'sender@example.com', firstName: 'Sender', lastName: '' },
|
||||||
|
to: [{ email: 'recipient@example.com', firstName: 'Recipient', lastName: '' }],
|
||||||
|
subject: 'Multipart Test Email',
|
||||||
|
content: 'This is the HTML version',
|
||||||
|
incoming: true
|
||||||
|
}),
|
||||||
|
[]
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should handle HTML email with inline images correctly', async () => {
|
||||||
|
// HTML content with embedded image reference
|
||||||
|
const htmlWithImage =
|
||||||
|
'<html><body><p>Test with image:</p><img src="cid:image1@example.com" alt="Test Image"></body></html>'
|
||||||
|
|
||||||
|
// Create image attachment
|
||||||
|
const imageAttachment = {
|
||||||
|
headers: [
|
||||||
|
['Content-Type', 'image/jpeg'],
|
||||||
|
['Content-Disposition', 'inline; filename="image.jpg"'],
|
||||||
|
['Content-ID', '<image1@example.com>']
|
||||||
|
],
|
||||||
|
content: 'base64encodedcontent' // Would normally be a Base64 string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create multipart message with HTML and image
|
||||||
|
const multipartMessage = {
|
||||||
|
envelope: {
|
||||||
|
from: { address: 'sender@example.com' },
|
||||||
|
to: [{ address: 'recipient@example.com' }]
|
||||||
|
},
|
||||||
|
message: {
|
||||||
|
headers: [
|
||||||
|
['Content-Type', 'multipart/related; boundary="boundary-string"'],
|
||||||
|
['Subject', 'Email with Inline Image'],
|
||||||
|
['From', 'Sender <sender@example.com>'],
|
||||||
|
['To', 'Recipient <recipient@example.com>']
|
||||||
|
],
|
||||||
|
contents: [
|
||||||
|
{
|
||||||
|
headers: [['Content-Type', 'text/html; charset=utf-8']],
|
||||||
|
content: htmlWithImage
|
||||||
|
},
|
||||||
|
imageAttachment
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
mockReq = {
|
||||||
|
headers: { 'x-hook-token': 'test-hook-token' },
|
||||||
|
body: multipartMessage
|
||||||
|
}
|
||||||
|
|
||||||
|
await handleMtaHook(mockReq as Request, mockRes as Response, mockCtx)
|
||||||
|
|
||||||
|
// Should process message with attachments
|
||||||
|
expect(createMessages).toHaveBeenCalledWith(
|
||||||
|
client.baseConfig,
|
||||||
|
mockCtx,
|
||||||
|
mockTxOperations,
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
client.mailServiceToken,
|
||||||
|
mockLoginInfo,
|
||||||
|
expect.objectContaining({
|
||||||
|
htmlContent: htmlWithImage
|
||||||
|
// Other fields as expected
|
||||||
|
}),
|
||||||
|
expect.arrayContaining([
|
||||||
|
expect.objectContaining({
|
||||||
|
contentType: 'image/jpeg',
|
||||||
|
name: 'image.jpg',
|
||||||
|
contentId: '<image1@example.com>'
|
||||||
|
// Other attachment fields
|
||||||
|
})
|
||||||
|
])
|
||||||
|
)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@ -12,14 +12,10 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
//
|
//
|
||||||
import { createHash, randomUUID } from 'crypto'
|
import { createHash } from 'crypto'
|
||||||
import { readEml, ReadedEmlJson } from 'eml-parse-js'
|
|
||||||
import { Request, Response } from 'express'
|
import { Request, Response } from 'express'
|
||||||
import TurndownService from 'turndown'
|
|
||||||
import sanitizeHtml from 'sanitize-html'
|
|
||||||
import { MeasureContext } from '@hcengineering/core'
|
import { MeasureContext } from '@hcengineering/core'
|
||||||
import {
|
import {
|
||||||
type Attachment,
|
|
||||||
type EmailContact,
|
type EmailContact,
|
||||||
type EmailMessage,
|
type EmailMessage,
|
||||||
createMessages,
|
createMessages,
|
||||||
@ -30,26 +26,8 @@ import { createRestTxOperations } from '@hcengineering/api-client'
|
|||||||
|
|
||||||
import { mailServiceToken, baseConfig, kvsClient } from './client'
|
import { mailServiceToken, baseConfig, kvsClient } from './client'
|
||||||
import config from './config'
|
import config from './config'
|
||||||
|
import { MtaMessage } from './types'
|
||||||
export interface MtaMessage {
|
import { getHeader, parseContent } from './utils'
|
||||||
envelope: {
|
|
||||||
from: {
|
|
||||||
address: string
|
|
||||||
}
|
|
||||||
to: {
|
|
||||||
address: string
|
|
||||||
}[]
|
|
||||||
}
|
|
||||||
message: {
|
|
||||||
headers: string[][]
|
|
||||||
contents: string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function getHeader (mta: MtaMessage, header: string): string | undefined {
|
|
||||||
const h = header.toLowerCase()
|
|
||||||
return mta.message.headers.find((header) => header[0].toLowerCase() === h)?.[1]?.trim()
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function handleMtaHook (req: Request, res: Response, ctx: MeasureContext): Promise<void> {
|
export async function handleMtaHook (req: Request, res: Response, ctx: MeasureContext): Promise<void> {
|
||||||
try {
|
try {
|
||||||
@ -142,75 +120,6 @@ export async function handleMtaHook (req: Request, res: Response, ctx: MeasureCo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function parseContent (
|
|
||||||
ctx: MeasureContext,
|
|
||||||
mta: MtaMessage
|
|
||||||
): Promise<{ content: string, attachments: Attachment[] }> {
|
|
||||||
const contentType = getHeader(mta, 'Content-Type')
|
|
||||||
if (contentType === undefined) {
|
|
||||||
throw new Error('Content-Type header not found')
|
|
||||||
}
|
|
||||||
|
|
||||||
if (contentType.toLowerCase().startsWith('text/plain')) {
|
|
||||||
return { content: mta.message.contents, attachments: [] }
|
|
||||||
}
|
|
||||||
|
|
||||||
const contents = `Content-Type: ${contentType}\r\n${mta.message.contents}`
|
|
||||||
const email = await new Promise<ReadedEmlJson>((resolve, reject) => {
|
|
||||||
readEml(contents, (err, json) => {
|
|
||||||
if (err !== undefined && err !== null) {
|
|
||||||
reject(err)
|
|
||||||
} else if (json === undefined) {
|
|
||||||
reject(new Error('Failed to parse email'))
|
|
||||||
} else {
|
|
||||||
resolve(json)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
let content = email.text ?? ''
|
|
||||||
let isMarkdown = false
|
|
||||||
if (email.html !== undefined) {
|
|
||||||
try {
|
|
||||||
const html = sanitizeHtml(email.html)
|
|
||||||
const tds = new TurndownService()
|
|
||||||
content = tds.turndown(html)
|
|
||||||
isMarkdown = true
|
|
||||||
} catch (error) {
|
|
||||||
ctx.warn('Failed to parse html content', { error })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const attachments: Attachment[] = []
|
|
||||||
if (config.storageConfig !== undefined) {
|
|
||||||
for (const a of email.attachments ?? []) {
|
|
||||||
if (a.name === undefined || a.name.length === 0) {
|
|
||||||
// EML parser returns attachments with empty name for parts of content
|
|
||||||
// that do not have "Content-Disposition: attachment" e.g. for part
|
|
||||||
// Content-Type: text/calendar; charset="UTF-8"; method=REQUEST
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
const attachment: Attachment = {
|
|
||||||
id: randomUUID(),
|
|
||||||
name: a.name,
|
|
||||||
data: Buffer.from(a.data64, 'base64'),
|
|
||||||
contentType: a.contentType.split(';')[0].trim()
|
|
||||||
}
|
|
||||||
attachments.push(attachment)
|
|
||||||
|
|
||||||
// For inline images, replace the CID references with the blob id
|
|
||||||
if (isMarkdown && a.inline && a.id !== undefined) {
|
|
||||||
const cid = a.id.replace(/[<>]/g, '')
|
|
||||||
content = content.replaceAll(
|
|
||||||
new RegExp(`!\\[.*?\\]\\(cid:${cid}\\)`, 'g'),
|
|
||||||
``
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return { content, attachments }
|
|
||||||
}
|
|
||||||
|
|
||||||
function getEmailContact (email: string): EmailContact {
|
function getEmailContact (email: string): EmailContact {
|
||||||
const parts = stripTags(email).split('@')
|
const parts = stripTags(email).split('@')
|
||||||
return {
|
return {
|
||||||
|
|||||||
29
services/mail/pod-inbound-mail/src/types.ts
Normal file
29
services/mail/pod-inbound-mail/src/types.ts
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
//
|
||||||
|
// Copyright © 2025 Hardcore Engineering Inc.
|
||||||
|
//
|
||||||
|
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License. You may
|
||||||
|
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
//
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
//
|
||||||
|
|
||||||
|
export interface MtaMessage {
|
||||||
|
envelope: {
|
||||||
|
from: {
|
||||||
|
address: string
|
||||||
|
}
|
||||||
|
to: {
|
||||||
|
address: string
|
||||||
|
}[]
|
||||||
|
}
|
||||||
|
message: {
|
||||||
|
headers: string[][]
|
||||||
|
contents: string
|
||||||
|
}
|
||||||
|
}
|
||||||
135
services/mail/pod-inbound-mail/src/utils.ts
Normal file
135
services/mail/pod-inbound-mail/src/utils.ts
Normal file
@ -0,0 +1,135 @@
|
|||||||
|
//
|
||||||
|
// Copyright © 2025 Hardcore Engineering Inc.
|
||||||
|
//
|
||||||
|
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License. You may
|
||||||
|
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
//
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
//
|
||||||
|
import { randomUUID } from 'crypto'
|
||||||
|
import { readEml, ReadedEmlJson } from 'eml-parse-js'
|
||||||
|
import TurndownService from 'turndown'
|
||||||
|
import sanitizeHtml from 'sanitize-html'
|
||||||
|
import { MeasureContext } from '@hcengineering/core'
|
||||||
|
import {
|
||||||
|
type Attachment
|
||||||
|
} from '@hcengineering/mail-common'
|
||||||
|
|
||||||
|
import { MtaMessage } from './types'
|
||||||
|
import config from './config'
|
||||||
|
|
||||||
|
export async function parseContent (
|
||||||
|
ctx: MeasureContext,
|
||||||
|
mta: MtaMessage
|
||||||
|
): Promise<{ content: string, attachments: Attachment[] }> {
|
||||||
|
const contentType = getHeader(mta, 'Content-Type')
|
||||||
|
if (contentType === undefined) {
|
||||||
|
throw new Error('Content-Type header not found')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (contentType.toLowerCase().startsWith('text/plain')) {
|
||||||
|
return { content: mta.message.contents, attachments: [] }
|
||||||
|
}
|
||||||
|
|
||||||
|
const email = await getEmailContent(mta.message.contents)
|
||||||
|
|
||||||
|
let content = email.text ?? ''
|
||||||
|
console.log('Content:', content)
|
||||||
|
let isMarkdown = false
|
||||||
|
if (email.html !== undefined) {
|
||||||
|
try {
|
||||||
|
const html = sanitizeHtml(email.html)
|
||||||
|
const tds = new TurndownService()
|
||||||
|
content = tds.turndown(html)
|
||||||
|
console.log('HTML Content:', content)
|
||||||
|
|
||||||
|
isMarkdown = true
|
||||||
|
} catch (error) {
|
||||||
|
ctx.warn('Failed to parse html content', { error })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const attachments: Attachment[] = []
|
||||||
|
if (config.storageConfig !== undefined) {
|
||||||
|
for (const a of email.attachments ?? []) {
|
||||||
|
if (a.name === undefined || a.name.length === 0) {
|
||||||
|
// EML parser returns attachments with empty name for parts of content
|
||||||
|
// that do not have "Content-Disposition: attachment" e.g. for part
|
||||||
|
// Content-Type: text/calendar; charset="UTF-8"; method=REQUEST
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const attachment: Attachment = {
|
||||||
|
id: randomUUID(),
|
||||||
|
name: a.name,
|
||||||
|
data: Buffer.from(a.data64, 'base64'),
|
||||||
|
contentType: a.contentType.split(';')[0].trim()
|
||||||
|
}
|
||||||
|
attachments.push(attachment)
|
||||||
|
|
||||||
|
// For inline images, replace the CID references with the blob id
|
||||||
|
if (isMarkdown && a.inline && a.id !== undefined) {
|
||||||
|
const cid = a.id.replace(/[<>]/g, '')
|
||||||
|
content = content.replaceAll(
|
||||||
|
new RegExp(`!\\[.*?\\]\\(cid:${cid}\\)`, 'g'),
|
||||||
|
``
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { content, attachments }
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getHeader (mta: MtaMessage, header: string): string | undefined {
|
||||||
|
const h = header.toLowerCase()
|
||||||
|
return mta.message.headers.find((header) => header[0].toLowerCase() === h)?.[1]?.trim()
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getEmailContent (mtaContent: string): Promise<ReadedEmlJson> {
|
||||||
|
if (mtaContent == null) {
|
||||||
|
return {
|
||||||
|
text: '',
|
||||||
|
html: '',
|
||||||
|
attachments: []
|
||||||
|
} as any
|
||||||
|
}
|
||||||
|
const contentRegex = /Content-Type/i
|
||||||
|
const content = contentRegex.test(mtaContent)
|
||||||
|
? mtaContent
|
||||||
|
: `Content-Type: ${guessContentType(mtaContent)}\r\n${mtaContent}`
|
||||||
|
const email = await new Promise<ReadedEmlJson>((resolve, reject) => {
|
||||||
|
readEml(content, (err, json) => {
|
||||||
|
if (err !== undefined && err !== null) {
|
||||||
|
reject(new Error(`Email parsing error: ${err.message}`))
|
||||||
|
} else if (json === undefined) {
|
||||||
|
reject(new Error('Email parser returned undefined result'))
|
||||||
|
} else {
|
||||||
|
resolve(json)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
if (isEmptyString(email.text) && isEmptyString(email.html)) {
|
||||||
|
return {
|
||||||
|
...email,
|
||||||
|
text: mtaContent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return email
|
||||||
|
}
|
||||||
|
|
||||||
|
function guessContentType (content: string): string {
|
||||||
|
// Simple heuristic - if it contains HTML tags, it's likely HTML
|
||||||
|
if (/<[a-z][\s\S]*>/i.test(content)) {
|
||||||
|
return 'text/html; charset="UTF-8"'
|
||||||
|
}
|
||||||
|
return 'text/plain; charset="UTF-8"'
|
||||||
|
}
|
||||||
|
|
||||||
|
function isEmptyString (str: string | undefined): boolean {
|
||||||
|
return str == null || str.trim() === ''
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user