UBERF-9634 Handle unsupported markdown in github integration (port to develop) (#8260)
Some checks are pending
CI / build (push) Waiting to run
CI / svelte-check (push) Blocked by required conditions
CI / formatting (push) Blocked by required conditions
CI / test (push) Blocked by required conditions
CI / uitest (push) Waiting to run
CI / uitest-pg (push) Waiting to run
CI / uitest-qms (push) Waiting to run
CI / uitest-workspaces (push) Waiting to run
CI / docker-build (push) Blocked by required conditions
CI / dist-build (push) Blocked by required conditions

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
This commit is contained in:
Alexander Onnikov 2025-03-18 19:20:24 +07:00 committed by GitHub
parent 31b54bb039
commit 6f63a407dd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 203 additions and 45 deletions

View File

@ -0,0 +1,38 @@
<!--
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
-->
<script lang="ts">
import { MarkupNode } from '@hcengineering/text'
import { Html } from '@hcengineering/ui'
export let node: MarkupNode
export let preview = false
function escapeHtml (unsafe: string): string {
return unsafe
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#039;')
}
$: content = node.content ?? []
$: value = escapeHtml(content.map((node) => node.text).join('/n'))
$: margin = preview ? '0' : null
</script>
{#if node}
<pre class="proseCodeBlock" style:margin><code><Html {value} /></code></pre>
{/if}

View File

@ -18,6 +18,7 @@
import CodeBlockNode from './CodeBlockNode.svelte'
import ObjectNode from './ObjectNode.svelte'
import MarkdownNode from './MarkdownNode.svelte'
import Node from './Node.svelte'
export let node: MarkupNode
@ -198,6 +199,8 @@
{/each}
{/if}
</th>
{:else if node.type === MarkupNodeType.markdown}
<MarkdownNode {node} {preview} />
{:else if node.type === MarkupNodeType.mermaid}
<!-- TODO -->
{:else if node.type === MarkupNodeType.comment}

View File

@ -39,7 +39,8 @@ export enum MarkupNodeType {
table_cell = 'tableCell',
table_header = 'tableHeader',
mermaid = 'mermaid',
comment = 'comment'
comment = 'comment',
markdown = 'markdown'
}
/** @public */

View File

@ -74,7 +74,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -107,7 +107,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -168,7 +168,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -233,7 +233,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -336,7 +336,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -453,7 +453,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -528,7 +528,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',
@ -692,7 +692,7 @@ Lorem ipsum dolor sit amet.
content: [
{
type: 'heading',
attrs: { level: 1 },
attrs: { level: 1, marker: '#' },
content: [
{
type: 'text',

View File

@ -445,7 +445,7 @@ const tokensBlock: Record<string, ParsingBlockRule> = {
},
heading: {
block: MarkupNodeType.heading,
getAttrs: (tok: Token) => ({ level: Number(tok.tag.slice(1)) })
getAttrs: (tok: Token) => ({ level: Number(tok.tag.slice(1)), marker: tok.markup })
},
code_block: {
block: MarkupNodeType.code_block,

View File

@ -104,8 +104,18 @@ export const storeNodes: Record<string, NodeProcessor> = {
},
heading: (state, node) => {
const attrs = nodeAttrs(node)
state.write(state.repeat('#', attrs.level !== undefined ? Number(attrs.level) : 1) + ' ')
state.renderInline(node)
if (attrs.marker === '=' && attrs.level === 1) {
state.renderInline(node)
state.ensureNewLine()
state.write('===\n')
} else if (attrs.marker === '-' && attrs.level === 2) {
state.renderInline(node)
state.ensureNewLine()
state.write('---\n')
} else {
state.write(state.repeat('#', attrs.level !== undefined ? Number(attrs.level) : 1) + ' ')
state.renderInline(node)
}
state.closeBlock(node)
},
horizontalRule: (state, node) => {
@ -228,6 +238,10 @@ export const storeNodes: Record<string, NodeProcessor> = {
')'
)
},
markdown: (state, node) => {
state.renderInline(node)
state.closeBlock(node)
},
comment: (state, node) => {
state.write('<!--')
state.renderInline(node)

View File

@ -21,23 +21,25 @@ import TableHeader from '@tiptap/extension-table-header'
import TableRow from '@tiptap/extension-table-row'
import TaskItem from '@tiptap/extension-task-item'
import TaskList from '@tiptap/extension-task-list'
import TextAlign from '@tiptap/extension-text-align'
import TextStyle from '@tiptap/extension-text-style'
import { CodeExtension, codeOptions } from '../marks/code'
import { BackgroundColor, TextColor } from '../marks/colors'
import { InlineCommentMark } from '../marks/inlineComment'
import { NoteBaseExtension } from '../marks/noteBase'
import { NodeUuid } from '../marks/nodeUuid'
import { CodeBlockExtension, codeBlockOptions } from '../nodes'
import { CommentNode } from '../nodes/comment'
import { FileNode, FileOptions } from '../nodes/file'
import { ImageNode, ImageOptions } from '../nodes/image'
import { MarkdownNode } from '../nodes/markdown'
import { MermaidExtension, mermaidOptions } from '../nodes/mermaid'
import { ReferenceNode } from '../nodes/reference'
import { TodoItemNode, TodoListNode } from '../nodes/todo'
import { CodeBlockExtension, codeBlockOptions } from '../nodes'
import { DefaultKit, DefaultKitOptions } from './default-kit'
import { CodeExtension, codeOptions } from '../marks/code'
import { NoteBaseExtension } from '../marks/noteBase'
import { CommentNode } from '../nodes/comment'
import { MermaidExtension, mermaidOptions } from '../nodes/mermaid'
import TextAlign from '@tiptap/extension-text-align'
import TextStyle from '@tiptap/extension-text-style'
import { BackgroundColor, TextColor } from '../marks/colors'
import { InlineCommentMark } from '../marks/inlineComment'
const headingLevels: Level[] = [1, 2, 3, 4, 5, 6]
@ -102,6 +104,7 @@ export const ServerKit = Extension.create<ServerKitOptions>({
TodoListNode,
ReferenceNode,
CommentNode,
MarkdownNode,
NodeUuid,
NoteBaseExtension,
TextStyle.configure({}),

View File

@ -19,4 +19,5 @@ export * from './todo'
export * from './file'
export * from './codeblock'
export * from './comment'
export * from './markdown'
export { getDataAttribute } from './utils'

View File

@ -0,0 +1,42 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { mergeAttributes, Node } from '@tiptap/core'
export const MarkdownNode = Node.create({
name: 'markdown',
group: 'block',
content: 'text*',
marks: '',
code: true,
defining: true,
parseHTML () {
return [
{
tag: 'pre[data-type="markdown"]',
preserveWhitespace: 'full'
}
]
},
renderHTML ({ node, HTMLAttributes }) {
return [
'pre',
mergeAttributes({ 'data-type': this.name }, this.options.HTMLAttributes, HTMLAttributes),
['code', {}, 0]
]
}
})

View File

@ -19,7 +19,9 @@ import {
BackgroundColor,
CodeExtension,
codeOptions,
CommentNode,
InlineCommentMark,
MarkdownNode,
TextColor,
TextStyle
} from '@hcengineering/text'
@ -158,7 +160,7 @@ async function buildEditorKit (): Promise<Extension<EditorKitOptions, any>> {
.then((kitExtensionCreators) => {
resolve(
Extension.create<EditorKitOptions>({
name: 'defaultKit',
name: 'editorKit',
addExtensions () {
const mode: TextEditorMode = this.options.mode ?? 'full'
@ -193,7 +195,16 @@ async function buildEditorKit (): Promise<Extension<EditorKitOptions, any>> {
[150, InlineCommentMark.configure({})],
[200, CodeBlockHighlighExtension.configure(codeBlockHighlightOptions)],
[210, CodeExtension.configure(codeOptions)],
[220, HardBreakExtension.configure({ shortcuts: mode })]
[220, HardBreakExtension.configure({ shortcuts: mode })],
[230, CommentNode],
[
240,
MarkdownNode.configure({
HTMLAttributes: {
class: 'proseCodeBlock'
}
})
]
]
if (this.options.submit !== false) {

View File

@ -99,7 +99,7 @@ const config: Config = (() => {
WebhookSecret: process.env[envMap.WebhookSecret] ?? 'secret',
EnterpriseHostname: process.env[envMap.EnterpriseHostname],
Port: parseInt(process.env[envMap.Port] ?? '3500'),
BotName: process.env[envMap.BotName] ?? 'dev[bot]',
BotName: process.env[envMap.BotName] ?? 'ao-huly-dev[bot]',
MongoURL: process.env[envMap.MongoURL],
ConfigurationDB: process.env[envMap.ConfigurationDB] ?? '%github',

View File

@ -545,13 +545,13 @@ A list of closed updated issues`
})
it('Check underline heading rule', () => {
const t1 = 'Hello\n---\nSome text'
const t1 = 'Hello\n---\n\nSome text'
const msg = parseMessageMarkdown(t1, 'ref://', 'http://', 'http://')
expect(msg.type).toEqual(MarkupNodeType.doc)
const md = serializeMessage(msg, 'ref://', 'http://')
expect(md).toEqual('## Hello\n\nSome text')
expect(md).toEqual('Hello\n---\n\nSome text')
})
it('Check horizontal line', () => {

View File

@ -188,7 +188,7 @@ export class CommentSyncManager implements DocSyncManager {
})
const messageData: MessageData = {
message: await this.provider.getMarkup(integration, event.comment.body)
message: await this.provider.getMarkupSafe(integration, event.comment.body)
}
if (commentData !== undefined) {
@ -280,7 +280,7 @@ export class CommentSyncManager implements DocSyncManager {
const account = existing?.modifiedBy ?? (await this.provider.getAccountU(comment.user))?._id ?? core.account.System
const messageData: MessageData = {
message: await this.provider.getMarkup(container.container, comment.body)
message: await this.provider.getMarkupSafe(container.container, comment.body)
}
if (existing === undefined) {
try {

View File

@ -796,7 +796,7 @@ export abstract class IssueSyncManagerBase {
// if (k === 'description' && pv != null) {
// const mdown = await this.provider.getMarkdown(pv)
// pv = await this.provider.getMarkup(container.container, mdown, this.stripGuestLink)
// pv = await this.provider.getMarkupSafe(container.container, mdown, this.stripGuestLink)
// }
// if (pv != null && pv !== v) {
// // We have conflict of values, assume platform is more proper one.
@ -1014,7 +1014,7 @@ export abstract class IssueSyncManagerBase {
// if (platformUpdate.description != null) {
// // Need to convert to markdown
// issueUpdate.body = await this.provider.getMarkdown(platformUpdate.description ?? '')
// issueData.description = await this.provider.getMarkup(
// issueData.description = await this.provider.getMarkupSafe(
// container.container,
// issueUpdate.body ?? '',
// this.stripGuestLink

View File

@ -220,7 +220,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
const update: IssueUpdate = {}
const du: DocumentUpdate<DocSyncInfo> = {}
if (event.changes.body !== undefined) {
update.description = await this.provider.getMarkup(integration, event.issue.body, this.stripGuestLink)
update.description = await this.provider.getMarkupSafe(integration, event.issue.body, this.stripGuestLink)
du.markdown = await this.provider.getMarkdown(update.description)
}
if (event.changes.title !== undefined) {
@ -413,7 +413,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
addHulyLink: false, // Do not need, since we create comment on Github about issue is connected.
current: {
title: issueExternal.title,
description: await this.provider.getMarkup(container.container, issueExternal.body, this.stripGuestLink)
description: await this.provider.getMarkupSafe(container.container, issueExternal.body, this.stripGuestLink)
}
}
needCreateConnectedAtHuly = true
@ -501,7 +501,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
const issueData = {
title: issueExternal.title,
description: await this.provider.getMarkup(container.container, issueExternal.body, this.stripGuestLink),
description: await this.provider.getMarkupSafe(container.container, issueExternal.body, this.stripGuestLink),
assignee: assignees[0]?.person,
repository: info.repository,
remainingTime: 0
@ -767,7 +767,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
},
{ url: issueExternal.url, id: existing._id }
)
issueData.description = await this.provider.getMarkup(container.container, body, this.stripGuestLink)
issueData.description = await this.provider.getMarkupSafe(container.container, body, this.stripGuestLink)
} else if (hasFieldStateChanges) {
await this.ctx.withLog(
'==> updateIssue',

View File

@ -174,7 +174,7 @@ export class ProjectsSyncManager implements DocSyncManager {
const messageData: MilestoneData = {
label: milestoneExternal.label,
description: await this.provider.getMarkup(container.container, milestoneExternal.description)
description: await this.provider.getMarkupSafe(container.container, milestoneExternal.description)
}
await this.handleDiffUpdateMilestone(existing, info, messageData, container, milestoneExternal)

View File

@ -211,7 +211,11 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
update.title = event.pull_request.title
}
if (event.changes.body !== undefined) {
update.description = await this.provider.getMarkup(integration, event.pull_request.body, this.stripGuestLink)
update.description = await this.provider.getMarkupSafe(
integration,
event.pull_request.body,
this.stripGuestLink
)
du.markdown = await this.provider.getMarkdown(update.description)
}
if (event.changes.base !== undefined) {
@ -464,7 +468,11 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
}
const pullRequestData: GithubPullRequestData = {
title: pullRequestExternal.title,
description: await this.provider.getMarkup(container.container, pullRequestExternal.body, this.stripGuestLink),
description: await this.provider.getMarkupSafe(
container.container,
pullRequestExternal.body,
this.stripGuestLink
),
assignee: assignees[0]?.person ?? null,
reviewers: reviewers.map((it: any) => it.person),
draft: pullRequestExternal.isDraft,
@ -1067,7 +1075,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
},
{ url: issueExternal.url }
)
issueData.description = await this.provider.getMarkup(container.container, body, this.stripGuestLink)
issueData.description = await this.provider.getMarkupSafe(container.container, body, this.stripGuestLink)
} else if (hasFieldsUpdate) {
await this.ctx.withLog('==> updatePullRequest:', {}, async () => {
this.ctx.info('update-fields', {

View File

@ -245,7 +245,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
)
if (reviewObj !== undefined) {
const lastModified = Date.now()
const body = await this.provider.getMarkup(integration, event.comment.body)
const body = await this.provider.getMarkupSafe(integration, event.comment.body)
await derivedClient.diffUpdate(
reviewData,
{
@ -343,7 +343,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
}
const messageData: ReviewCommentData = {
body: await this.provider.getMarkup(container.container, reviewComment.body),
body: await this.provider.getMarkupSafe(container.container, reviewComment.body),
diffHunk: reviewComment.diffHunk,
isMinimized: reviewComment.isMinimized,
reviewUrl: reviewComment.pullRequestReview.url,
@ -414,7 +414,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
if (Object.keys(platformUpdate).length > 0) {
if (platformUpdate.body !== undefined) {
const body = await this.provider.getMarkup(container.container, platformUpdate.body)
const body = await this.provider.getMarkupSafe(container.container, platformUpdate.body)
const okit = (await this.provider.getOctokit(account)) ?? container.container.octokit
const q = `mutation updateReviewComment($commentID: ID!, $body: String!) {
updatePullRequestReviewComment(input: {

View File

@ -305,7 +305,7 @@ export class ReviewSyncManager implements DocSyncManager {
const account = existing?.modifiedBy ?? (await this.provider.getAccount(review.author))?._id ?? core.account.System
const messageData: ReviewData = {
body: await this.provider.getMarkup(container.container, review.body),
body: await this.provider.getMarkupSafe(container.container, review.body),
state: toReviewState(review.state),
comments: (review.comments?.nodes ?? []).map((it) => it.url)
}

View File

@ -87,6 +87,11 @@ export interface IntegrationManager {
getAccount: (user?: UserInfo | null) => Promise<any | undefined>
getAccountU: (user: User) => Promise<any | undefined>
getOctokit: (account: PersonId) => Promise<Octokit | undefined>
getMarkupSafe: (
container: IntegrationContainer,
text?: string | null,
preprocessor?: (nodes: MarkupNode) => Promise<void>
) => Promise<string>
getMarkup: (
container: IntegrationContainer,
text?: string | null,

View File

@ -55,7 +55,7 @@ import { LiveQuery } from '@hcengineering/query'
import { StorageAdapter } from '@hcengineering/server-core'
import { getPublicLinkUrl } from '@hcengineering/server-guest-resources'
import task, { ProjectType, TaskType } from '@hcengineering/task'
import { MarkupNode, jsonToMarkup } from '@hcengineering/text'
import { MarkupNode, MarkupNodeType, jsonToMarkup } from '@hcengineering/text'
import { isMarkdownsEquals } from '@hcengineering/text-markdown'
import tracker from '@hcengineering/tracker'
import { User } from '@octokit/webhooks-types'
@ -174,8 +174,40 @@ export class GithubWorker implements IntegrationManager {
body: string
): Promise<{ markdownCompatible: boolean, markdown: string }> {
const markupText = await this.getMarkup(container, body)
const markDown = await this.getMarkdown(markupText)
return { markdownCompatible: isMarkdownsEquals(body, markDown), markdown: markDown }
const markdown = await this.getMarkdown(markupText)
const markdownCompatible = isMarkdownsEquals(body, markdown)
return { markdownCompatible, markdown }
}
async getMarkupSafe (
container: IntegrationContainer,
text?: string | null,
preprocessor?: (nodes: MarkupNode) => Promise<void>
): Promise<string> {
if (text == null) {
return ''
}
const markup = await this.getMarkup(container, text, preprocessor)
const markdown = await this.getMarkdown(markup)
const compatible = isMarkdownsEquals(text, markdown)
return compatible
? markup
: jsonToMarkup({
type: MarkupNodeType.doc,
content: [
{
type: MarkupNodeType.markdown,
content: [
{
type: MarkupNodeType.text,
text
}
]
}
]
})
}
async getMarkup (