UBERF-9634 Handle unsupported markdown in github integration (#8246)

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
This commit is contained in:
Alexander Onnikov 2025-03-17 18:55:01 +07:00 committed by GitHub
parent ead12940b5
commit 30fcd1bffd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 270 additions and 45 deletions

View File

@ -0,0 +1,38 @@
<!--
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
-->
<script lang="ts">
import { MarkupNode } from '@hcengineering/text'
import { Html } from '@hcengineering/ui'
export let node: MarkupNode
export let preview = false
function escapeHtml (unsafe: string): string {
return unsafe
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#039;')
}
$: content = node.content ?? []
$: value = escapeHtml(content.map((node) => node.text).join('/n'))
$: margin = preview ? '0' : null
</script>
{#if node}
<pre class="proseCodeBlock" style:margin><code><Html {value} /></code></pre>
{/if}

View File

@ -18,6 +18,7 @@
import CodeBlockNode from './CodeBlockNode.svelte'
import ObjectNode from './ObjectNode.svelte'
import MarkdownNode from './MarkdownNode.svelte'
import Node from './Node.svelte'
export let node: MarkupNode
@ -186,6 +187,8 @@
{/each}
{/if}
</th>
{:else if node.type === MarkupNodeType.markdown}
<MarkdownNode {node} {preview} />
{:else if node.type === MarkupNodeType.mermaid}
<!-- TODO -->
{:else if node.type === MarkupNodeType.comment}

View File

@ -39,7 +39,8 @@ export enum MarkupNodeType {
table_cell = 'tableCell',
table_header = 'tableHeader',
mermaid = 'mermaid',
comment = 'comment'
comment = 'comment',
markdown = 'markdown'
}
/** @public */

View File

@ -21,23 +21,25 @@ import TableHeader from '@tiptap/extension-table-header'
import TableRow from '@tiptap/extension-table-row'
import TaskItem from '@tiptap/extension-task-item'
import TaskList from '@tiptap/extension-task-list'
import TextAlign from '@tiptap/extension-text-align'
import TextStyle from '@tiptap/extension-text-style'
import { CodeExtension, codeOptions } from '../marks/code'
import { BackgroundColor, TextColor } from '../marks/colors'
import { InlineCommentMark } from '../marks/inlineComment'
import { NoteBaseExtension } from '../marks/noteBase'
import { NodeUuid } from '../marks/nodeUuid'
import { CodeBlockExtension, codeBlockOptions } from '../nodes'
import { CommentNode } from '../nodes/comment'
import { FileNode, FileOptions } from '../nodes/file'
import { ImageNode, ImageOptions } from '../nodes/image'
import { MarkdownNode } from '../nodes/markdown'
import { MermaidExtension, mermaidOptions } from '../nodes/mermaid'
import { ReferenceNode } from '../nodes/reference'
import { TodoItemNode, TodoListNode } from '../nodes/todo'
import { CodeBlockExtension, codeBlockOptions } from '../nodes'
import { DefaultKit, DefaultKitOptions } from './default-kit'
import { CodeExtension, codeOptions } from '../marks/code'
import { NoteBaseExtension } from '../marks/noteBase'
import { CommentNode } from '../nodes/comment'
import { MermaidExtension, mermaidOptions } from '../nodes/mermaid'
import TextAlign from '@tiptap/extension-text-align'
import TextStyle from '@tiptap/extension-text-style'
import { BackgroundColor, TextColor } from '../marks/colors'
import { InlineCommentMark } from '../marks/inlineComment'
const headingLevels: Level[] = [1, 2, 3, 4, 5, 6]
@ -102,6 +104,7 @@ export const ServerKit = Extension.create<ServerKitOptions>({
TodoListNode,
ReferenceNode,
CommentNode,
MarkdownNode,
NodeUuid,
NoteBaseExtension,
TextStyle.configure({}),

View File

@ -41,6 +41,70 @@ const tests: Array<{ name: string, markdown: string, markup: object }> = [
]
}
},
{
name: 'heading',
markdown: '# heading 1',
markup: {
type: 'doc',
content: [
{
type: 'heading',
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
text: 'heading 1',
marks: []
}
]
}
]
}
},
{
name: 'block heading level 1',
markdown: `heading 1
===
`,
markup: {
type: 'doc',
content: [
{
type: 'heading',
attrs: { level: 1, marker: '=' },
content: [
{
type: 'text',
text: 'heading 1',
marks: []
}
]
}
]
}
},
{
name: 'block heading level 2',
markdown: `heading 2
---
`,
markup: {
type: 'doc',
content: [
{
type: 'heading',
attrs: { level: 2, marker: '-' },
content: [
{
type: 'text',
text: 'heading 2',
marks: []
}
]
}
]
}
},
{
name: 'text with heading',
markdown: `# Lorem ipsum
@ -52,7 +116,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -85,7 +149,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -146,7 +210,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -211,7 +275,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -314,7 +378,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -431,7 +495,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -506,7 +570,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -670,7 +734,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',

View File

@ -446,7 +446,10 @@ const tokensBlock: Record<string, ParsingBlockRule> = {
},
heading: {
block: MarkupNodeType.heading,
getAttrs: (tok: Token) => ({ level: Number(tok.tag.slice(1)) })
getAttrs: (tok: Token) => ({
level: Number(tok.tag.slice(1)),
marker: tok.markup
})
},
code_block: {
block: MarkupNodeType.code_block,

View File

@ -88,8 +88,18 @@ export const storeNodes: Record<string, NodeProcessor> = {
},
heading: (state, node) => {
const attrs = nodeAttrs(node)
state.write(state.repeat('#', attrs.level !== undefined ? Number(attrs.level) : 1) + ' ')
state.renderInline(node)
if (attrs.marker === '=' && attrs.level === 1) {
state.renderInline(node)
state.ensureNewLine()
state.write('===\n')
} else if (attrs.marker === '-' && attrs.level === 2) {
state.renderInline(node)
state.ensureNewLine()
state.write('---\n')
} else {
state.write(state.repeat('#', attrs.level !== undefined ? Number(attrs.level) : 1) + ' ')
state.renderInline(node)
}
state.closeBlock(node)
},
horizontalRule: (state, node) => {
@ -212,6 +222,10 @@ export const storeNodes: Record<string, NodeProcessor> = {
')'
)
},
markdown: (state, node) => {
state.renderInline(node)
state.closeBlock(node)
},
comment: (state, node) => {
state.write('<!--')
state.renderInline(node)

View File

@ -19,4 +19,5 @@ export * from './todo'
export * from './file'
export * from './codeblock'
export * from './comment'
export * from './markdown'
export { getDataAttribute } from './utils'

View File

@ -0,0 +1,42 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { mergeAttributes, Node } from '@tiptap/core'
export const MarkdownNode = Node.create({
name: 'markdown',
group: 'block',
content: 'text*',
marks: '',
code: true,
defining: true,
parseHTML () {
return [
{
tag: 'pre[data-type="markdown"]',
preserveWhitespace: 'full'
}
]
},
renderHTML ({ node, HTMLAttributes }) {
return [
'pre',
mergeAttributes({ 'data-type': this.name }, this.options.HTMLAttributes, HTMLAttributes),
['code', {}, 0]
]
}
})

View File

@ -19,7 +19,9 @@ import {
BackgroundColor,
CodeExtension,
codeOptions,
CommentNode,
InlineCommentMark,
MarkdownNode,
TextColor,
TextStyle
} from '@hcengineering/text'
@ -158,7 +160,7 @@ async function buildEditorKit (): Promise<Extension<EditorKitOptions, any>> {
.then((kitExtensionCreators) => {
resolve(
Extension.create<EditorKitOptions>({
name: 'defaultKit',
name: 'editorKit',
addExtensions () {
const mode: TextEditorMode = this.options.mode ?? 'full'
@ -193,7 +195,16 @@ async function buildEditorKit (): Promise<Extension<EditorKitOptions, any>> {
[150, InlineCommentMark.configure({})],
[200, CodeBlockHighlighExtension.configure(codeBlockHighlightOptions)],
[210, CodeExtension.configure(codeOptions)],
[220, HardBreakExtension.configure({ shortcuts: mode })]
[220, HardBreakExtension.configure({ shortcuts: mode })],
[230, CommentNode],
[
240,
MarkdownNode.configure({
HTMLAttributes: {
class: 'proseCodeBlock'
}
})
]
]
if (this.options.submit !== false) {

View File

@ -99,7 +99,7 @@ const config: Config = (() => {
WebhookSecret: process.env[envMap.WebhookSecret] ?? 'secret',
EnterpriseHostname: process.env[envMap.EnterpriseHostname],
Port: parseInt(process.env[envMap.Port] ?? '3500'),
BotName: process.env[envMap.BotName] ?? 'dev[bot]',
BotName: process.env[envMap.BotName] ?? 'ao-huly-dev[bot]',
MongoURL: process.env[envMap.MongoURL],
ConfigurationDB: process.env[envMap.ConfigurationDB] ?? '%github',

View File

@ -545,13 +545,13 @@ A list of closed updated issues`
})
it('Check underline heading rule', () => {
const t1 = 'Hello\n---\nSome text'
const t1 = 'Hello\n---\n\nSome text'
const msg = parseMessageMarkdown(t1, 'ref://', 'http://', 'http://')
expect(msg.type).toEqual(MarkupNodeType.doc)
const md = serializeMessage(msg, 'ref://', 'http://')
expect(md).toEqual('## Hello\n\nSome text')
expect(md).toEqual(t1)
})
it('Check horizontal line', () => {

View File

@ -189,7 +189,7 @@ export class CommentSyncManager implements DocSyncManager {
})
const messageData: MessageData = {
message: await this.provider.getMarkup(integration, event.comment.body)
message: await this.provider.getMarkupSafe(integration, event.comment.body)
}
if (commentData !== undefined) {
@ -281,7 +281,7 @@ export class CommentSyncManager implements DocSyncManager {
const account = existing?.modifiedBy ?? (await this.provider.getAccountU(comment.user))?._id ?? core.account.System
const messageData: MessageData = {
message: await this.provider.getMarkup(container.container, comment.body)
message: await this.provider.getMarkupSafe(container.container, comment.body)
}
if (existing === undefined) {
try {

View File

@ -792,7 +792,7 @@ export abstract class IssueSyncManagerBase {
if (k === 'description' && pv != null) {
const mdown = await this.provider.getMarkdown(pv)
pv = await this.provider.getMarkup(container.container, mdown, this.stripGuestLink)
pv = await this.provider.getMarkupSafe(container.container, mdown, this.stripGuestLink)
}
if (pv != null && pv !== v) {
// We have conflict of values, assume platform is more proper one.
@ -1008,7 +1008,7 @@ export abstract class IssueSyncManagerBase {
if (platformUpdate.description != null) {
// Need to convert to markdown
issueUpdate.body = await this.provider.getMarkdown(platformUpdate.description ?? '')
issueData.description = await this.provider.getMarkup(
issueData.description = await this.provider.getMarkupSafe(
container.container,
issueUpdate.body ?? '',
this.stripGuestLink

View File

@ -219,7 +219,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
const update: IssueUpdate = {}
const du: DocumentUpdate<DocSyncInfo> = {}
if (event.changes.body !== undefined) {
update.description = await this.provider.getMarkup(integration, event.issue.body, this.stripGuestLink)
update.description = await this.provider.getMarkupSafe(integration, event.issue.body, this.stripGuestLink)
du.markdown = await this.provider.getMarkdown(update.description)
}
if (event.changes.title !== undefined) {
@ -412,7 +412,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
addHulyLink: false, // Do not need, since we create comment on Github about issue is connected.
current: {
title: issueExternal.title,
description: await this.provider.getMarkup(container.container, issueExternal.body, this.stripGuestLink)
description: await this.provider.getMarkupSafe(container.container, issueExternal.body, this.stripGuestLink)
}
}
needCreateConnectedAtHuly = true
@ -500,7 +500,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
const issueData = {
title: issueExternal.title,
description: await this.provider.getMarkup(container.container, issueExternal.body, this.stripGuestLink),
description: await this.provider.getMarkupSafe(container.container, issueExternal.body, this.stripGuestLink),
assignee: assignees[0]?.person,
repository: info.repository,
remainingTime: 0
@ -767,7 +767,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
},
{ url: issueExternal.url, id: existing._id }
)
issueData.description = await this.provider.getMarkup(container.container, body, this.stripGuestLink)
issueData.description = await this.provider.getMarkupSafe(container.container, body, this.stripGuestLink)
} else if (hasFieldStateChanges) {
await this.ctx.withLog(
'==> updateIssue',

View File

@ -171,7 +171,7 @@ export class ProjectsSyncManager implements DocSyncManager {
const messageData: MilestoneData = {
label: milestoneExternal.label,
description: await this.provider.getMarkup(container.container, milestoneExternal.description)
description: await this.provider.getMarkupSafe(container.container, milestoneExternal.description)
}
await this.handleDiffUpdateMilestone(existing, info, messageData, container, milestoneExternal)

View File

@ -210,7 +210,11 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
update.title = event.pull_request.title
}
if (event.changes.body !== undefined) {
update.description = await this.provider.getMarkup(integration, event.pull_request.body, this.stripGuestLink)
update.description = await this.provider.getMarkupSafe(
integration,
event.pull_request.body,
this.stripGuestLink
)
du.markdown = await this.provider.getMarkdown(update.description)
}
if (event.changes.base !== undefined) {
@ -462,7 +466,11 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
}
const pullRequestData: GithubPullRequestData = {
title: pullRequestExternal.title,
description: await this.provider.getMarkup(container.container, pullRequestExternal.body, this.stripGuestLink),
description: await this.provider.getMarkupSafe(
container.container,
pullRequestExternal.body,
this.stripGuestLink
),
assignee: assignees[0]?.person ?? null,
reviewers: reviewers.map((it) => it.person),
draft: pullRequestExternal.isDraft,
@ -1066,7 +1074,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
},
{ url: issueExternal.url }
)
issueData.description = await this.provider.getMarkup(container.container, body, this.stripGuestLink)
issueData.description = await this.provider.getMarkupSafe(container.container, body, this.stripGuestLink)
} else if (hasFieldsUpdate) {
await this.ctx.withLog('==> updatePullRequest:', {}, async () => {
this.ctx.info('update-fields', {

View File

@ -246,7 +246,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
)
if (reviewObj !== undefined) {
const lastModified = Date.now()
const body = await this.provider.getMarkup(integration, event.comment.body)
const body = await this.provider.getMarkupSafe(integration, event.comment.body)
await derivedClient.diffUpdate(
reviewData,
{
@ -344,7 +344,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
}
const messageData: ReviewCommentData = {
body: await this.provider.getMarkup(container.container, reviewComment.body),
body: await this.provider.getMarkupSafe(container.container, reviewComment.body),
diffHunk: reviewComment.diffHunk,
isMinimized: reviewComment.isMinimized,
reviewUrl: reviewComment.pullRequestReview.url,
@ -415,7 +415,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
if (Object.keys(platformUpdate).length > 0) {
if (platformUpdate.body !== undefined) {
const body = await this.provider.getMarkup(container.container, platformUpdate.body)
const body = await this.provider.getMarkupSafe(container.container, platformUpdate.body)
const okit = (await this.provider.getOctokit(account as Ref<PersonAccount>)) ?? container.container.octokit
const q = `mutation updateReviewComment($commentID: ID!, $body: String!) {
updatePullRequestReviewComment(input: {

View File

@ -306,7 +306,7 @@ export class ReviewSyncManager implements DocSyncManager {
const account = existing?.modifiedBy ?? (await this.provider.getAccount(review.author))?._id ?? core.account.System
const messageData: ReviewData = {
body: await this.provider.getMarkup(container.container, review.body),
body: await this.provider.getMarkupSafe(container.container, review.body),
state: toReviewState(review.state),
comments: (review.comments?.nodes ?? []).map((it) => it.url)
}

View File

@ -86,6 +86,11 @@ export interface IntegrationManager {
getAccount: (user?: UserInfo | null) => Promise<PersonAccount | undefined>
getAccountU: (user: User) => Promise<PersonAccount | undefined>
getOctokit: (account: Ref<PersonAccount>) => Promise<Octokit | undefined>
getMarkupSafe: (
container: IntegrationContainer,
text?: string | null,
preprocessor?: (nodes: MarkupNode) => Promise<void>
) => Promise<string>
getMarkup: (
container: IntegrationContainer,
text?: string | null,

View File

@ -52,7 +52,7 @@ import { LiveQuery } from '@hcengineering/query'
import { StorageAdapter } from '@hcengineering/server-core'
import { getPublicLinkUrl } from '@hcengineering/server-guest-resources'
import task, { ProjectType, TaskType } from '@hcengineering/task'
import { MarkupNode, isMarkdownsEquals, jsonToMarkup } from '@hcengineering/text'
import { MarkupNode, MarkupNodeType, isMarkdownsEquals, jsonToMarkup } from '@hcengineering/text'
import tracker from '@hcengineering/tracker'
import { User } from '@octokit/webhooks-types'
import { App, Octokit } from 'octokit'
@ -161,8 +161,40 @@ export class GithubWorker implements IntegrationManager {
body: string
): Promise<{ markdownCompatible: boolean, markdown: string }> {
const markupText = await this.getMarkup(container, body)
const markDown = await this.getMarkdown(markupText)
return { markdownCompatible: isMarkdownsEquals(body, markDown), markdown: markDown }
const markdown = await this.getMarkdown(markupText)
const markdownCompatible = isMarkdownsEquals(body, markdown)
return { markdownCompatible, markdown }
}
async getMarkupSafe (
container: IntegrationContainer,
text?: string | null,
preprocessor?: (nodes: MarkupNode) => Promise<void>
): Promise<string> {
if (text == null) {
return ''
}
const markup = await this.getMarkup(container, text, preprocessor)
const markdown = await this.getMarkdown(markup)
const compatible = isMarkdownsEquals(text, markdown)
return compatible
? markup
: jsonToMarkup({
type: MarkupNodeType.doc,
content: [
{
type: MarkupNodeType.markdown,
content: [
{
type: MarkupNodeType.text,
text
}
]
}
]
})
}
async getMarkup (