wieerwill/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2025-05-12 10:25:51 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge b25bcadf51 into a6e491edf6

Browse Source
This commit is contained in:
Anna Khismatullina 2025-05-07 13:19:59 +07:00 committed by GitHub
commit cdda9ec451
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 353 additions and 333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -4,6 +4,7 @@ _api-extractor-temp/
temp/ temp/
.idea .idea
pods/workspace/init/ pods/workspace/init/
pods/workspace/init-scripts/
# Logs # Logs
*.log *.log

8
.vscode/launch.json vendored
View File

@ -279,9 +279,11 @@
"MODEL_VERSION": "v0.7.1", "MODEL_VERSION": "v0.7.1",
"WS_OPERATION": "all+backup", "WS_OPERATION": "all+backup",
"BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin", "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
"BACKUP_BUCKET": "dev-backups" "BACKUP_BUCKET": "dev-backups",
// "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init", "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init",
// "INIT_WORKSPACE": "staging-dev" "INIT_WORKSPACE": "staging-dev",
"QUEUE_CONFIG": "huly.local:19092",
"QUEUE_REGION": ""
}, },
"runtimeVersion": "20", "runtimeVersion": "20",
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],

17
packages/importer/src/huly/cards.ts
View File

@ -32,11 +32,10 @@ import * as yaml from 'js-yaml'
import { contentType } from 'mime-types' import { contentType } from 'mime-types'
import * as path from 'path' import * as path from 'path'
import { IntlString } from '../../../platform/types' import { IntlString } from '../../../platform/types'
import { Props, UnifiedDoc, UnifiedUpdate, UnifiedFile, UnifiedMixin } from '../types'
import { MetadataRegistry, AssociationMetadata } from './registry'
import { readMarkdownContent, readYamlHeader } from './parsing'
import { Logger } from '../importer/logger' import { Logger } from '../importer/logger'
import { validateSchema } from './validation' import { Props, UnifiedDoc, UnifiedFile, UnifiedMixin, UnifiedUpdate } from '../types'
import { UnifiedFormatParser } from './parser'
import { AssociationMetadata, MetadataRegistry } from './registry'
import { import {
AssociationSchema, AssociationSchema,
BaseFieldType, BaseFieldType,
@ -51,6 +50,7 @@ import {
StringFieldType, StringFieldType,
TagSchema TagSchema
} from './schema' } from './schema'
import { validateSchema } from './validation'
export interface UnifiedDocProcessResult { export interface UnifiedDocProcessResult {
docs: Map<string, Array<UnifiedDoc<Doc>>> docs: Map<string, Array<UnifiedDoc<Doc>>>
@ -62,6 +62,7 @@ export interface UnifiedDocProcessResult {
export class CardsProcessor { export class CardsProcessor {
constructor ( constructor (
private readonly metadataRegistry: MetadataRegistry, private readonly metadataRegistry: MetadataRegistry,
private readonly parser: UnifiedFormatParser,
private readonly logger: Logger private readonly logger: Logger
) {} ) {}
@ -199,7 +200,7 @@ export class CardsProcessor {
for (const entry of entries) { for (const entry of entries) {
if (entry.isFile() && entry.name.endsWith('.md')) { if (entry.isFile() && entry.name.endsWith('.md')) {
const cardPath = path.join(currentPath, entry.name) const cardPath = path.join(currentPath, entry.name)
const { class: cardType, ...cardProps } = await readYamlHeader(cardPath) const { class: cardType, ...cardProps } = this.parser.readYamlHeader(cardPath)
if (masterTagId !== undefined) { if (masterTagId !== undefined) {
await this.processCard(result, cardPath, cardProps, masterTagId, masterTagAssociaions, masterTagAttributes) await this.processCard(result, cardPath, cardProps, masterTagId, masterTagAssociaions, masterTagAttributes)
@ -231,7 +232,7 @@ export class CardsProcessor {
for (const entry of entries) { for (const entry of entries) {
if (entry.isFile() && entry.name.endsWith('.md')) { if (entry.isFile() && entry.name.endsWith('.md')) {
const cardPath = path.join(currentDir, entry.name) const cardPath = path.join(currentDir, entry.name)
const { class: cardType, ...cardProps } = await readYamlHeader(cardPath) const { class: cardType, ...cardProps } = this.parser.readYamlHeader(cardPath)
if (cardType !== undefined && cardType.startsWith('card:types:') === false) { if (cardType !== undefined && cardType.startsWith('card:types:') === false) {
throw new Error('Unsupported card type: ' + cardType + ' in ' + cardPath) throw new Error('Unsupported card type: ' + cardType + ' in ' + cardPath)
@ -310,7 +311,7 @@ export class CardsProcessor {
for (const entry of entries) { for (const entry of entries) {
const childCardPath = path.join(cardDir, entry.name) const childCardPath = path.join(cardDir, entry.name)
const { class: cardClass, ...cardProps } = await readYamlHeader(childCardPath) const { class: cardClass, ...cardProps } = this.parser.readYamlHeader(childCardPath)
await this.processCard( await this.processCard(
result, result,
childCardPath, childCardPath,
@ -594,7 +595,7 @@ export class CardsProcessor {
{ {
_class: masterTagId, _class: masterTagId,
collabField: 'content', collabField: 'content',
contentProvider: () => readMarkdownContent(cardPath), contentProvider: () => Promise.resolve(this.parser.readMarkdownContent(cardPath)),
props: cardProps as Props<Card> props: cardProps as Props<Card>
}, },
...relations ...relations

346
packages/importer/src/huly/huly.ts
View File

@ -15,7 +15,7 @@
/* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable @typescript-eslint/no-unused-vars */
import { type Attachment } from '@hcengineering/attachment' import { type Attachment } from '@hcengineering/attachment'
import card from '@hcengineering/card' import card from '@hcengineering/card'
import contact, { Employee, type Person, SocialIdentity } from '@hcengineering/contact' import contact, { Employee, type Person } from '@hcengineering/contact'
import documents, { import documents, {
ControlledDocument, ControlledDocument,
DocumentCategory, DocumentCategory,
@ -24,18 +24,14 @@ import documents, {
} from '@hcengineering/controlled-documents' } from '@hcengineering/controlled-documents'
import { import {
AccountUuid, AccountUuid,
type Class,
type Doc,
generateId, generateId,
PersonId, PersonId,
type Ref, type Ref,
SocialIdType, SocialIdType,
type Space,
type TxOperations type TxOperations
} from '@hcengineering/core' } from '@hcengineering/core'
import document, { type Document } from '@hcengineering/document' import document, { type Document } from '@hcengineering/document'
import core from '@hcengineering/model-core' import core from '@hcengineering/model-core'
import { MarkupMarkType, type MarkupNode, MarkupNodeType, traverseNode, traverseNodeMarks } from '@hcengineering/text'
import tracker, { type Issue, Project } from '@hcengineering/tracker' import tracker, { type Issue, Project } from '@hcengineering/tracker'
import * as fs from 'fs' import * as fs from 'fs'
import sizeOf from 'image-size' import sizeOf from 'image-size'
@ -59,11 +55,11 @@ import {
WorkspaceImporter WorkspaceImporter
} from '../importer/importer' } from '../importer/importer'
import { type Logger } from '../importer/logger' import { type Logger } from '../importer/logger'
import { BaseMarkdownPreprocessor } from '../importer/preprocessor'
import { type FileUploader } from '../importer/uploader' import { type FileUploader } from '../importer/uploader'
import { CardsProcessor } from './cards' import { CardsProcessor } from './cards'
import { MetadataRegistry, MentionMetadata } from './registry' import { UnifiedFormatParser } from './parser'
import { readMarkdownContent, readYamlHeader } from './parsing' import { HulyMarkdownPreprocessor, type AttachmentMetadata } from './preprocessor'
import { MetadataRegistry } from './registry'
export interface HulyComment { export interface HulyComment {
author: string author: string
text: string text: string
@ -165,175 +161,10 @@ export interface HulyOrgSpaceSettings extends HulySpaceSettings {
qara?: string qara?: string
} }
class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor {
constructor (
private readonly urlProvider: (id: string) => string,
private readonly logger: Logger,
private readonly metadataRegistry: MetadataRegistry,
private readonly attachMetaByPath: Map<string, AttachmentMetadata>,
personsByName: Map<string, Ref<Person>>
) {
super(personsByName)
}
process (json: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): MarkupNode {
traverseNode(json, (node) => {
if (node.type === MarkupNodeType.image) {
this.processImageNode(node, id, spaceId)
} else {
this.processLinkMarks(node, id, spaceId)
this.processMentions(node)
}
return true
})
return json
}
private processImageNode (node: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): void {
const src = node.attrs?.src
if (src === undefined) return
const sourcePath = this.getSourcePath(id)
if (sourcePath == null) return
const href = decodeURI(src as string)
const fullPath = path.resolve(path.dirname(sourcePath), href)
const attachmentMeta = this.attachMetaByPath.get(fullPath)
if (attachmentMeta === undefined) {
this.logger.error(`Attachment image not found for ${fullPath}`)
return
}
if (!this.metadataRegistry.hasRefMetadata(sourcePath)) {
this.logger.error(`Source metadata not found for ${sourcePath}`)
return
}
const sourceMeta = this.metadataRegistry.getRefMetadata(sourcePath)
this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta)
this.alterImageNode(node, attachmentMeta.id, attachmentMeta.name)
}
private processLinkMarks (node: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): void {
traverseNodeMarks(node, (mark) => {
if (mark.type !== MarkupMarkType.link) return
const sourcePath = this.getSourcePath(id)
if (sourcePath == null) return
const href = decodeURI(mark.attrs?.href ?? '')
const fullPath = path.resolve(path.dirname(sourcePath), href)
if (this.metadataRegistry.hasRefMetadata(fullPath)) {
const targetDocMeta = this.metadataRegistry.getRefMetadata(fullPath)
this.alterMentionNode(node, targetDocMeta)
} else if (this.attachMetaByPath.has(fullPath)) {
const attachmentMeta = this.attachMetaByPath.get(fullPath)
if (attachmentMeta !== undefined) {
this.alterAttachmentLinkNode(node, attachmentMeta)
if (this.metadataRegistry.hasRefMetadata(sourcePath)) {
const sourceMeta = this.metadataRegistry.getRefMetadata(sourcePath)
this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta)
}
}
} else {
this.logger.log('Unknown link type, leave it as is: ' + href)
}
})
}
private alterImageNode (node: MarkupNode, id: string, name: string): void {
node.type = MarkupNodeType.image
if (node.attrs !== undefined) {
node.attrs = {
'file-id': id,
src: this.urlProvider(id),
width: node.attrs.width ?? null,
height: node.attrs.height ?? null,
align: node.attrs.align ?? null,
alt: name,
title: name
}
const mimeType = this.getContentType(name)
if (mimeType !== undefined) {
node.attrs['data-file-type'] = mimeType
}
}
}
private alterMentionNode (node: MarkupNode, targetMeta: MentionMetadata): void {
node.type = MarkupNodeType.reference
node.attrs = {
id: targetMeta.id,
label: targetMeta.refTitle,
objectclass: targetMeta.class,
text: '',
content: ''
}
}
private alterAttachmentLinkNode (node: MarkupNode, targetMeta: AttachmentMetadata): void {
const stats = fs.statSync(targetMeta.path)
node.type = MarkupNodeType.file
node.attrs = {
'file-id': targetMeta.id,
'data-file-name': targetMeta.name,
'data-file-size': stats.size,
'data-file-href': targetMeta.path
}
const mimeType = this.getContentType(targetMeta.name)
if (mimeType !== undefined) {
node.attrs['data-file-type'] = mimeType
}
}
private getContentType (fileName: string): string | undefined {
const mimeType = contentType(fileName)
return mimeType !== false ? mimeType : undefined
}
private getSourcePath (id: Ref<Doc>): string | null {
const sourcePath = this.metadataRegistry.getPath(id)
if (sourcePath === undefined) {
this.logger.error(`Source file path not found for ${id}`)
return null
}
return sourcePath
}
private updateAttachmentMetadata (
fullPath: string,
attachmentMeta: AttachmentMetadata,
id: Ref<Doc>,
spaceId: Ref<Space>,
sourceMeta: MentionMetadata
): void {
this.attachMetaByPath.set(fullPath, {
...attachmentMeta,
spaceId,
parentId: id,
parentClass: sourceMeta.class as Ref<Class<Doc<Space>>>
})
}
}
interface AttachmentMetadata {
id: Ref<Attachment>
name: string
path: string
parentId?: Ref<Doc>
parentClass?: Ref<Class<Doc<Space>>>
spaceId?: Ref<Space>
}
export class HulyFormatImporter { export class HulyFormatImporter {
private readonly importerEmailPlaceholder = 'newuser@huly.io' private readonly personsByName = new Map<string, Ref<Person>>()
private readonly importerNamePlaceholder = 'New User'
private personsByName = new Map<string, Ref<Person>>()
private employeesByName = new Map<string, Ref<Employee>>() private employeesByName = new Map<string, Ref<Employee>>()
private accountsByEmail = new Map<string, AccountUuid>() private readonly accountsByName = new Map<string, AccountUuid>()
private readonly personIdByEmail = new Map<string, PersonId>() private readonly personIdByEmail = new Map<string, PersonId>()
private controlledDocumentCategories = new Map<string, Ref<DocumentCategory>>() private controlledDocumentCategories = new Map<string, Ref<DocumentCategory>>()
@ -341,20 +172,20 @@ export class HulyFormatImporter {
private readonly metadataRegistry = new MetadataRegistry() private readonly metadataRegistry = new MetadataRegistry()
private readonly cardsProcessor: CardsProcessor private readonly cardsProcessor: CardsProcessor
private readonly parser: UnifiedFormatParser
constructor ( constructor (
private readonly client: TxOperations, private readonly client: TxOperations,
private readonly fileUploader: FileUploader, private readonly fileUploader: FileUploader,
private readonly logger: Logger, private readonly logger: Logger,
private readonly importerSocialId?: PersonId, variables?: Record<string, any>
private readonly importerPerson?: Ref<Person>
) { ) {
this.cardsProcessor = new CardsProcessor(this.metadataRegistry, this.logger) this.parser = new UnifiedFormatParser(variables ?? {})
this.cardsProcessor = new CardsProcessor(this.metadataRegistry, this.parser, this.logger)
} }
private async initCaches (): Promise<void> { private async initCaches (): Promise<void> {
await this.cachePersonsByNames() await this.cachePersonsByNames()
await this.cacheAccountsByEmails()
await this.cacheEmployeesByName() await this.cacheEmployeesByName()
await this.cacheControlledDocumentCategories() await this.cacheControlledDocumentCategories()
} }
@ -485,7 +316,7 @@ export class HulyFormatImporter {
try { try {
this.logger.log(`Processing ${spaceName}...`) this.logger.log(`Processing ${spaceName}...`)
const spaceConfig = yaml.load(fs.readFileSync(yamlPath, 'utf8')) as HulySpaceSettings const spaceConfig = this.parser.readYaml(yamlPath) as HulySpaceSettings
if (spaceConfig?.class === undefined) { if (spaceConfig?.class === undefined) {
this.logger.error(`Skipping ${spaceName}: not a space - no class specified`) this.logger.error(`Skipping ${spaceName}: not a space - no class specified`)
@ -560,7 +391,7 @@ export class HulyFormatImporter {
for (const issueFile of issueFiles) { for (const issueFile of issueFiles) {
const issuePath = path.join(currentPath, issueFile) const issuePath = path.join(currentPath, issueFile)
const issueHeader = (await readYamlHeader(issuePath)) as HulyIssueHeader const issueHeader = this.parser.readYamlHeader(issuePath) as HulyIssueHeader
if (issueHeader.class === undefined) { if (issueHeader.class === undefined) {
this.logger.error(`Skipping ${issueFile}: not an issue`) this.logger.error(`Skipping ${issueFile}: not an issue`)
@ -578,7 +409,7 @@ export class HulyFormatImporter {
class: tracker.class.Issue, class: tracker.class.Issue,
title: issueHeader.title, title: issueHeader.title,
number: parseInt(issueNumber ?? 'NaN'), number: parseInt(issueNumber ?? 'NaN'),
descrProvider: async () => await readMarkdownContent(issuePath), descrProvider: () => Promise.resolve(this.parser.readMarkdownContent(issuePath)),
status: { name: issueHeader.status }, status: { name: issueHeader.status },
priority: issueHeader.priority, priority: issueHeader.priority,
estimation: issueHeader.estimation, estimation: issueHeader.estimation,
@ -606,9 +437,6 @@ export class HulyFormatImporter {
return undefined return undefined
} }
if (name === this.importerNamePlaceholder && this.importerPerson != null) {
return this.importerPerson
}
const person = this.personsByName.get(name) const person = this.personsByName.get(name)
if (person === undefined) { if (person === undefined) {
throw new Error(`Person not found: ${name}`) throw new Error(`Person not found: ${name}`)
@ -617,10 +445,6 @@ export class HulyFormatImporter {
} }
private async getPersonIdByEmail (email: string): Promise<PersonId> { private async getPersonIdByEmail (email: string): Promise<PersonId> {
if (email === this.importerEmailPlaceholder && this.importerSocialId != null) {
return this.importerSocialId
}
const personId = this.personIdByEmail.get(email) const personId = this.personIdByEmail.get(email)
if (personId !== undefined) { if (personId !== undefined) {
return personId return personId
@ -640,10 +464,10 @@ export class HulyFormatImporter {
return socialId._id return socialId._id
} }
private findAccountByEmail (email: string): AccountUuid { private findAccountByName (name: string): AccountUuid {
const account = this.accountsByEmail.get(email) const account = this.accountsByName.get(name)
if (account === undefined) { if (account === undefined) {
throw new Error(`Account not found: ${email}`) throw new Error(`Account not found: ${name}`)
} }
return account return account
} }
@ -666,7 +490,7 @@ export class HulyFormatImporter {
for (const docFile of docFiles) { for (const docFile of docFiles) {
const docPath = path.join(currentPath, docFile) const docPath = path.join(currentPath, docFile)
const docHeader = (await readYamlHeader(docPath)) as HulyDocumentHeader const docHeader = this.parser.readYamlHeader(docPath) as HulyDocumentHeader
if (docHeader.class === undefined) { if (docHeader.class === undefined) {
this.logger.error(`Skipping ${docFile}: not a document`) this.logger.error(`Skipping ${docFile}: not a document`)
@ -680,7 +504,7 @@ export class HulyFormatImporter {
id: this.metadataRegistry.getRef(docPath) as Ref<Document>, id: this.metadataRegistry.getRef(docPath) as Ref<Document>,
class: document.class.Document, class: document.class.Document,
title: docHeader.title, title: docHeader.title,
descrProvider: async () => await readMarkdownContent(docPath), descrProvider: () => Promise.resolve(this.parser.readMarkdownContent(docPath)),
subdocs: [] // Will be added via builder subdocs: [] // Will be added via builder
} }
@ -707,7 +531,7 @@ export class HulyFormatImporter {
for (const docFile of docFiles) { for (const docFile of docFiles) {
const docPath = path.join(currentPath, docFile) const docPath = path.join(currentPath, docFile)
const docHeader = (await readYamlHeader(docPath)) as HulyControlledDocumentHeader | HulyDocumentTemplateHeader const docHeader = this.parser.readYamlHeader(docPath) as HulyControlledDocumentHeader | HulyDocumentTemplateHeader
if (docHeader.class === undefined) { if (docHeader.class === undefined) {
this.logger.error(`Skipping ${docFile}: not a document`) this.logger.error(`Skipping ${docFile}: not a document`)
@ -788,55 +612,51 @@ export class HulyFormatImporter {
) )
} }
private async processProject (projectHeader: HulyProjectSettings): Promise<ImportProject> { private async processProject (data: HulyProjectSettings): Promise<ImportProject> {
return { return {
class: tracker.class.Project, class: tracker.class.Project,
id: projectHeader.id as Ref<Project>, id: data.id as Ref<Project>,
title: projectHeader.title, title: data.title,
identifier: projectHeader.identifier, identifier: data.identifier,
private: projectHeader.private ?? false, private: data.private ?? false,
autoJoin: projectHeader.autoJoin ?? true, autoJoin: data.autoJoin ?? true,
archived: projectHeader.archived ?? false, archived: data.archived ?? false,
description: projectHeader.description, description: data.description,
emoji: projectHeader.emoji, emoji: data.emoji,
defaultIssueStatus: defaultIssueStatus: data.defaultIssueStatus !== undefined ? { name: data.defaultIssueStatus } : undefined,
projectHeader.defaultIssueStatus !== undefined ? { name: projectHeader.defaultIssueStatus } : undefined, owners: data.owners !== undefined ? data.owners.map((name) => this.findAccountByName(name)) : [],
owners: members: data.members !== undefined ? data.members.map((name) => this.findAccountByName(name)) : [],
projectHeader.owners !== undefined ? projectHeader.owners.map((email) => this.findAccountByEmail(email)) : [],
members:
projectHeader.members !== undefined ? projectHeader.members.map((email) => this.findAccountByEmail(email)) : [],
docs: [] docs: []
} }
} }
private async processTeamspace (spaceHeader: HulyTeamspaceSettings): Promise<ImportTeamspace> { private async processTeamspace (data: HulyTeamspaceSettings): Promise<ImportTeamspace> {
return { return {
class: document.class.Teamspace, class: document.class.Teamspace,
title: spaceHeader.title, title: data.title,
private: spaceHeader.private ?? false, private: data.private ?? false,
autoJoin: spaceHeader.autoJoin ?? true, autoJoin: data.autoJoin ?? true,
archived: spaceHeader.archived ?? false, archived: data.archived ?? false,
description: spaceHeader.description, description: data.description,
emoji: spaceHeader.emoji, emoji: data.emoji,
owners: spaceHeader.owners !== undefined ? spaceHeader.owners.map((email) => this.findAccountByEmail(email)) : [], owners: data.owners !== undefined ? data.owners.map((name) => this.findAccountByName(name)) : [],
members: members: data.members !== undefined ? data.members.map((name) => this.findAccountByName(name)) : [],
spaceHeader.members !== undefined ? spaceHeader.members.map((email) => this.findAccountByEmail(email)) : [],
docs: [] docs: []
} }
} }
private async processOrgSpace (spaceHeader: HulyOrgSpaceSettings): Promise<ImportOrgSpace> { private async processOrgSpace (data: HulyOrgSpaceSettings): Promise<ImportOrgSpace> {
return { return {
class: documents.class.OrgSpace, class: documents.class.OrgSpace,
title: spaceHeader.title, title: data.title,
private: spaceHeader.private ?? false, private: data.private ?? false,
archived: spaceHeader.archived ?? false, archived: data.archived ?? false,
description: spaceHeader.description, description: data.description,
owners: spaceHeader.owners?.map((email) => this.findAccountByEmail(email)) ?? [], owners: data.owners?.map((name) => this.findAccountByName(name)) ?? [],
members: spaceHeader.members?.map((email) => this.findAccountByEmail(email)) ?? [], members: data.members?.map((name) => this.findAccountByName(name)) ?? [],
qualified: spaceHeader.qualified !== undefined ? this.findAccountByEmail(spaceHeader.qualified) : undefined, qualified: data.qualified !== undefined ? this.findAccountByName(data.qualified) : undefined,
manager: spaceHeader.manager !== undefined ? this.findAccountByEmail(spaceHeader.manager) : undefined, manager: data.manager !== undefined ? this.findAccountByName(data.manager) : undefined,
qara: spaceHeader.qara !== undefined ? this.findAccountByEmail(spaceHeader.qara) : undefined, qara: data.qara !== undefined ? this.findAccountByName(data.qara) : undefined,
docs: [] docs: []
} }
} }
@ -876,10 +696,10 @@ export class HulyFormatImporter {
author, author,
owner, owner,
abstract: header.abstract, abstract: header.abstract,
reviewers: header.reviewers?.map((email) => this.findEmployeeByName(email)) ?? [], reviewers: header.reviewers?.map((name) => this.findEmployeeByName(name)) ?? [],
approvers: header.approvers?.map((email) => this.findEmployeeByName(email)) ?? [], approvers: header.approvers?.map((name) => this.findEmployeeByName(name)) ?? [],
coAuthors: header.coAuthors?.map((email) => this.findEmployeeByName(email)) ?? [], coAuthors: header.coAuthors?.map((name) => this.findEmployeeByName(name)) ?? [],
descrProvider: async () => await readMarkdownContent(docPath), descrProvider: () => Promise.resolve(this.parser.readMarkdownContent(docPath)),
ccReason: header.changeControl?.reason, ccReason: header.changeControl?.reason,
ccImpact: header.changeControl?.impact, ccImpact: header.changeControl?.impact,
ccDescription: header.changeControl?.description, ccDescription: header.changeControl?.description,
@ -919,10 +739,10 @@ export class HulyFormatImporter {
author, author,
owner, owner,
abstract: header.abstract, abstract: header.abstract,
reviewers: header.reviewers?.map((email) => this.findEmployeeByName(email)) ?? [], reviewers: header.reviewers?.map((name) => this.findEmployeeByName(name)) ?? [],
approvers: header.approvers?.map((email) => this.findEmployeeByName(email)) ?? [], approvers: header.approvers?.map((name) => this.findEmployeeByName(name)) ?? [],
coAuthors: header.coAuthors?.map((email) => this.findEmployeeByName(email)) ?? [], coAuthors: header.coAuthors?.map((name) => this.findEmployeeByName(name)) ?? [],
descrProvider: async () => await readMarkdownContent(docPath), descrProvider: () => Promise.resolve(this.parser.readMarkdownContent(docPath)),
ccReason: header.changeControl?.reason, ccReason: header.changeControl?.reason,
ccImpact: header.changeControl?.impact, ccImpact: header.changeControl?.impact,
ccDescription: header.changeControl?.description, ccDescription: header.changeControl?.description,
@ -930,54 +750,14 @@ export class HulyFormatImporter {
} }
} }
private async cacheAccountsByEmails (): Promise<void> {
const employees = await this.client.findAll(
contact.mixin.Employee,
{ active: true },
{ lookup: { _id: { socialIds: contact.class.SocialIdentity } } }
)
this.accountsByEmail = employees.reduce((map, employee) => {
employee.$lookup?.socialIds?.forEach((socialId) => {
if ((socialId as SocialIdentity).type === SocialIdType.EMAIL) {
map.set((socialId as SocialIdentity).value, employee.personUuid)
}
})
return map
}, new Map())
}
private async cachePersonIdsByEmails (): Promise<void> {
const employees = await this.client.findAll(
contact.mixin.Employee,
{ active: true },
{ lookup: { _id: { socialIds: contact.class.SocialIdentity } } }
)
this.accountsByEmail = employees.reduce((map, employee) => {
employee.$lookup?.socialIds?.forEach((socialId) => {
if ((socialId as SocialIdentity).type === SocialIdType.EMAIL) {
map.set((socialId as SocialIdentity).value, employee.personUuid)
}
})
return map
}, new Map())
}
private async cachePersonsByNames (): Promise<void> { private async cachePersonsByNames (): Promise<void> {
this.personsByName = (await this.client.findAll(contact.class.Person, {})) (await this.client.findAll(contact.class.Person, {})).forEach((person) => {
.map((person) => { const name = person.name.split(',').reverse().join(' ')
return { this.personsByName.set(name, person._id)
_id: person._id, if (person.personUuid !== undefined) {
name: person.name.split(',').reverse().join(' ') this.accountsByName.set(name, person.personUuid as AccountUuid)
} }
}) })
.reduce((refByName, person) => {
refByName.set(person.name, person._id)
return refByName
}, new Map())
} }
private async cacheEmployeesByName (): Promise<void> { private async cacheEmployeesByName (): Promise<void> {

79
packages/importer/src/huly/parser.ts Normal file
View File

@ -0,0 +1,79 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import * as fs from 'fs'
import * as yaml from 'js-yaml'
const VARIABLE_REGEX = /\${\S+?}/
const YAML_HEADER_REGEX = /^---\n([\s\S]*?)\n---/
const MARKDOWN_CONTENT_REGEX = /^---\n[\s\S]*?\n---\n(.*)$/s
export class UnifiedFormatParser {
constructor (private readonly variables: Record<string, any>) {}
public readYaml (filePath: string): Record<string, any> {
const content = fs.readFileSync(filePath, 'utf8')
const data = yaml.load(content) as Record<string, any>
return this.resolveProps(data)
}
public readYamlHeader (filePath: string): Record<string, any> {
const content = fs.readFileSync(filePath, 'utf8')
const match = content.match(YAML_HEADER_REGEX)
if (match != null) {
const header = yaml.load(match[1])
return this.resolveProps(header as Record<string, any>)
}
return {}
}
public readMarkdownContent (filePath: string): string {
const content = fs.readFileSync(filePath, 'utf8')
const match = content.match(MARKDOWN_CONTENT_REGEX)
return match != null ? match[1] : content
}
public resolveProps (data: Record<string, any>): Record<string, any> {
for (const key in data) {
const value = (data as any)[key]
;(data as any)[key] = this.resolveValue(value)
}
return data
}
private resolveValue (value: any): any {
if (typeof value === 'object') {
if (Array.isArray(value)) {
return value.map((v) => this.resolveValue(v))
} else {
return this.resolveProps(value)
}
} else if (typeof value === 'string') {
while (true) {
const matched = VARIABLE_REGEX.exec(value)
if (matched === null) break
const result = this.variables[matched[0]]
if (result === undefined) {
throw new Error(`Variable ${matched[0]} not found`)
} else {
value = value.replaceAll(matched[0], result)
VARIABLE_REGEX.lastIndex = 0
}
}
return value
}
return value
}
}

32
packages/importer/src/huly/parsing.ts
View File

@ -1,32 +0,0 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import * as fs from 'fs'
import * as yaml from 'js-yaml'
export async function readYamlHeader (filePath: string): Promise<any> {
const content = fs.readFileSync(filePath, 'utf8')
const match = content.match(/^---\n([\s\S]*?)\n---/)
if (match != null) {
return yaml.load(match[1])
}
return {}
}
export async function readMarkdownContent (filePath: string): Promise<string> {
const content = fs.readFileSync(filePath, 'utf8')
const match = content.match(/^---\n[\s\S]*?\n---\n(.*)$/s)
return match != null ? match[1] : content
}

191
packages/importer/src/huly/preprocessor.ts Normal file
View File

@ -0,0 +1,191 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
import { Attachment } from '@hcengineering/attachment'
import { type Person } from '@hcengineering/contact'
import {
type Class,
type Doc,
type Ref,
type Space
} from '@hcengineering/core'
import { MarkupMarkType, type MarkupNode, MarkupNodeType, traverseNode, traverseNodeMarks } from '@hcengineering/text'
import * as fs from 'fs'
import { contentType } from 'mime-types'
import * as path from 'path'
import { type Logger } from '../importer/logger'
import { BaseMarkdownPreprocessor } from '../importer/preprocessor'
import { MentionMetadata, MetadataRegistry } from './registry'
export interface AttachmentMetadata {
id: Ref<Attachment>
name: string
path: string
parentId?: Ref<Doc>
parentClass?: Ref<Class<Doc<Space>>>
spaceId?: Ref<Space>
}
export class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor {
constructor (
private readonly urlProvider: (id: string) => string,
private readonly logger: Logger,
private readonly metadataRegistry: MetadataRegistry,
private readonly attachMetaByPath: Map<string, AttachmentMetadata>,
personsByName: Map<string, Ref<Person>>
) {
super(personsByName)
}
process (json: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): MarkupNode {
traverseNode(json, (node) => {
if (node.type === MarkupNodeType.image) {
this.processImageNode(node, id, spaceId)
} else {
this.processLinkMarks(node, id, spaceId)
this.processMentions(node)
}
return true
})
return json
}
private processImageNode (node: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): void {
const src = node.attrs?.src
if (src === undefined) return
const sourcePath = this.getSourcePath(id)
if (sourcePath == null) return
const href = decodeURI(src as string)
const fullPath = path.resolve(path.dirname(sourcePath), href)
const attachmentMeta = this.attachMetaByPath.get(fullPath)
if (attachmentMeta === undefined) {
this.logger.error(`Attachment image not found for ${fullPath}`)
return
}
if (!this.metadataRegistry.hasRefMetadata(sourcePath)) {
this.logger.error(`Source metadata not found for ${sourcePath}`)
return
}
const sourceMeta = this.metadataRegistry.getRefMetadata(sourcePath)
this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta)
this.alterImageNode(node, attachmentMeta.id, attachmentMeta.name)
}
private processLinkMarks (node: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): void {
traverseNodeMarks(node, (mark) => {
if (mark.type !== MarkupMarkType.link) return
const sourcePath = this.getSourcePath(id)
if (sourcePath == null) return
const href = decodeURI(mark.attrs?.href ?? '')
const fullPath = path.resolve(path.dirname(sourcePath), href)
if (this.metadataRegistry.hasRefMetadata(fullPath)) {
const targetDocMeta = this.metadataRegistry.getRefMetadata(fullPath)
this.alterMentionNode(node, targetDocMeta)
} else if (this.attachMetaByPath.has(fullPath)) {
const attachmentMeta = this.attachMetaByPath.get(fullPath)
if (attachmentMeta !== undefined) {
this.alterAttachmentLinkNode(node, attachmentMeta)
if (this.metadataRegistry.hasRefMetadata(sourcePath)) {
const sourceMeta = this.metadataRegistry.getRefMetadata(sourcePath)
this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta)
}
}
} else {
this.logger.log('Unknown link type, leave it as is: ' + href)
}
})
}
private alterImageNode (node: MarkupNode, id: string, name: string): void {
node.type = MarkupNodeType.image
if (node.attrs !== undefined) {
node.attrs = {
'file-id': id,
src: this.urlProvider(id),
width: node.attrs.width ?? null,
height: node.attrs.height ?? null,
align: node.attrs.align ?? null,
alt: name,
title: name
}
const mimeType = this.getContentType(name)
if (mimeType !== undefined) {
node.attrs['data-file-type'] = mimeType
}
}
}
private alterMentionNode (node: MarkupNode, targetMeta: MentionMetadata): void {
node.type = MarkupNodeType.reference
node.attrs = {
id: targetMeta.id,
label: targetMeta.refTitle,
objectclass: targetMeta.class,
text: '',
content: ''
}
}
private alterAttachmentLinkNode (node: MarkupNode, targetMeta: AttachmentMetadata): void {
const stats = fs.statSync(targetMeta.path)
node.type = MarkupNodeType.file
node.attrs = {
'file-id': targetMeta.id,
'data-file-name': targetMeta.name,
'data-file-size': stats.size,
'data-file-href': targetMeta.path
}
const mimeType = this.getContentType(targetMeta.name)
if (mimeType !== undefined) {
node.attrs['data-file-type'] = mimeType
}
}
private getContentType (fileName: string): string | undefined {
const mimeType = contentType(fileName)
return mimeType !== false ? mimeType : undefined
}
private getSourcePath (id: Ref<Doc>): string | null {
const sourcePath = this.metadataRegistry.getPath(id)
if (sourcePath === undefined) {
this.logger.error(`Source file path not found for ${id}`)
return null
}
return sourcePath
}
private updateAttachmentMetadata (
fullPath: string,
attachmentMeta: AttachmentMetadata,
id: Ref<Doc>,
spaceId: Ref<Space>,
sourceMeta: MentionMetadata
): void {
this.attachMetaByPath.set(fullPath, {
...attachmentMeta,
spaceId,
parentId: id,
parentClass: sourceMeta.class as Ref<Class<Doc<Space>>>
})
}
}

4
server/tool/src/index.ts
View File

@ -208,9 +208,9 @@ export async function initializeWorkspace (
progress: (value: number) => Promise<void> progress: (value: number) => Promise<void>
): Promise<void> { ): Promise<void> {
const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace) const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace)
const initRepoDir = getMetadata(toolPlugin.metadata.InitRepoDir) const initRepoDir = getMetadata(toolPlugin.metadata.InitRepoDir) ?? ''
ctx.info('Init script details', { initWS, initRepoDir }) ctx.info('Init script details', { initWS, initRepoDir })
if (initWS === undefined || initRepoDir === undefined) return // if (initWS === undefined || initRepoDir === undefined) return
const initScriptFile = path.resolve(initRepoDir, 'script.yaml') const initScriptFile = path.resolve(initRepoDir, 'script.yaml')
if (!fs.existsSync(initScriptFile)) { if (!fs.existsSync(initScriptFile)) {

4
server/tool/src/initializer.ts
View File

@ -205,9 +205,7 @@ export class WorkspaceInitializer {
try { try {
const uploader = new StorageFileUploader(this.ctx, this.storageAdapter, this.wsIds) const uploader = new StorageFileUploader(this.ctx, this.storageAdapter, this.wsIds)
const initPath = path.resolve(this.initRepoDir, step.path) const initPath = path.resolve(this.initRepoDir, step.path)
// eslint-disable-next-line no-template-curly-in-string const importer = new HulyFormatImporter(this.client, uploader, logger, vars)
const initPerson = vars[`\${${this.creatorPersonVar}}`]
const importer = new HulyFormatImporter(this.client, uploader, logger, this.socialId, initPerson)
await importer.importFolder(initPath) await importer.importFolder(initPath)
} catch (error) { } catch (error) {
logger.error('Import failed', error) logger.error('Import failed', error)