wieerwill/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2025-05-11 18:01:59 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Controlled documents import fixes (#8615)

Browse Source 
Signed-off-by: Anna Khismatullina <anna.khismatullina@gmail.com>
This commit is contained in:
Anna Khismatullina 2025-04-22 17:40:05 +07:00 committed by GitHub
parent 9693f937d8
commit 55543c72d3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 138 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
packages/importer/src/huly/cards.ts
View File

@ -81,20 +81,22 @@ export class CardsProcessor {
await this.processMetadata(directoryPath, result, topLevelTypes)
const typesRefs = topLevelTypes.map((type) => type.props._id) as Ref<MasterTag>[]
const updateDefaultSpace: UnifiedUpdate<CardSpace> = {
_class: card.class.CardSpace,
_id: 'card:space:Default' as Ref<CardSpace>,
space: core.space.Model,
props: {
$push: {
types: {
$each: [...new Set(typesRefs)],
$position: 0
if (typesRefs.length > 0) {
const updateDefaultSpace: UnifiedUpdate<CardSpace> = {
_class: card.class.CardSpace,
_id: 'card:space:Default' as Ref<CardSpace>,
space: core.space.Model,
props: {
$push: {
types: {
$each: [...new Set(typesRefs)],
$position: 0
}
}
}
}
result.updates.set('card:space:Default', [updateDefaultSpace])
}
result.updates.set('card:space:Default', [updateDefaultSpace])
await this.processSystemTypeCards(directoryPath, result, new Map(), new Map())
await this.processCards(directoryPath, result, new Map(), new Map())

23
packages/importer/src/huly/huly.ts
View File

@ -126,6 +126,7 @@ export interface HulyControlledDocumentHeader {
template: string
author: string
owner: string
category?: string
abstract?: string
reviewers?: string[]
approvers?: string[]
@ -320,6 +321,7 @@ export class HulyFormatImporter {
private personsByName = new Map<string, Ref<Person>>()
private accountsByEmail = new Map<string, Ref<PersonAccount>>()
private employeesByName = new Map<string, Ref<Employee>>()
private controlledDocumentCategories = new Map<string, Ref<DocumentCategory>>()
private readonly fileMetaByPath = new Map<string, AttachmentMetadata>()
@ -338,6 +340,7 @@ export class HulyFormatImporter {
await this.cachePersonsByNames()
await this.cacheAccountsByEmails()
await this.cacheEmployeesByName()
await this.cacheControlledDocumentCategories()
}
async importFolder (folderPath: string): Promise<void> {
@ -674,8 +677,8 @@ export class HulyFormatImporter {
throw new Error(`Unknown document class ${docHeader.class} in ${docFile}`)
}
const documentMetaId = this.metadataRegistry.getRef(docPath) as Ref<DocumentMeta>
this.metadataRegistry.setRefMetadata(docPath, documents.class.DocumentMeta, docHeader.title)
const documentMetaId = generateId<DocumentMeta>()
this.metadataRegistry.setRefMetadata(docPath, documents.class.DocumentMeta, docHeader.title, documentMetaId)
if (docHeader.class === documents.class.ControlledDocument) {
const doc = await this.processControlledDocument(
@ -814,7 +817,7 @@ export class HulyFormatImporter {
}
const templateId = this.metadataRegistry.getRef(templatePath) as Ref<ControlledDocument>
const category = header.category !== undefined ? this.controlledDocumentCategories.get(header.category) : undefined
return {
id,
metaId,
@ -825,6 +828,7 @@ export class HulyFormatImporter {
major: 0,
minor: 1,
state: DocumentState.Draft,
category,
author,
owner,
abstract: header.abstract,
@ -852,6 +856,7 @@ export class HulyFormatImporter {
}
const codeMatch = path.basename(docPath).match(/^\[([^\]]+)\]/)
const category = header.category !== undefined ? this.controlledDocumentCategories.get(header.category) : undefined
return {
id,
metaId,
@ -862,7 +867,7 @@ export class HulyFormatImporter {
major: 0,
minor: 1,
state: DocumentState.Draft,
category: header.category as Ref<DocumentCategory>,
category,
author,
owner,
abstract: header.abstract,
@ -913,6 +918,16 @@ export class HulyFormatImporter {
}, new Map())
}
private async cacheControlledDocumentCategories (): Promise<void> {
this.controlledDocumentCategories = (await this.client.findAll(documents.class.DocumentCategory, {})).reduce(
(refByCode, category) => {
refByCode.set(category.code, category._id)
return refByCode
},
new Map()
)
}
private async collectFileMetadata (folderPath: string): Promise<void> {
const processDir = async (dir: string): Promise<void> => {
const entries = fs.readdirSync(dir, { withFileTypes: true })

4
packages/importer/src/huly/registry.ts
View File

@ -106,8 +106,8 @@ export class MetadataRegistry {
this.pathToTagMetadata.set(tagPath, metadata)
}
public setRefMetadata (path: string, _class: string, title: string): void {
const ref = this.getRef(path)
public setRefMetadata (path: string, _class: string, title: string, id?: Ref<Doc>): void {
const ref = id ?? this.getRef(path)
this.pathToMentionMetadata.set(path, {
id: ref,
class: _class,

103
packages/importer/src/importer/builder.ts
View File

@ -20,7 +20,7 @@ import {
ImportControlledDocument,
ImportControlledDocumentTemplate,
ImportOrgSpace,
type ImportControlledDoc,
type ImportControlledDocOrTemplate,
type ImportDocument,
type ImportIssue,
type ImportProject,
@ -43,6 +43,8 @@ const MAX_PROJECT_IDENTIFIER_LENGTH = 5
const PROJECT_IDENTIFIER_REGEX = /^[a-zA-Z_][a-zA-Z0-9_]*$/
export class ImportWorkspaceBuilder {
private readonly projectTypes = new Map<string, ImportProjectType>()
private readonly projects = new Map<string, ImportProject>()
private readonly issuesByProject = new Map<string, Map<string, ImportIssue>>()
private readonly issueParents = new Map<string, string>()
@ -53,11 +55,15 @@ export class ImportWorkspaceBuilder {
private readonly qmsSpaces = new Map<string, ImportOrgSpace>()
private readonly qmsTemplates = new Map<Ref<ControlledDocument>, string>()
private readonly qmsDocsBySpace = new Map<string, Map<string, ImportControlledDoc>>()
private readonly qmsDocsBySpace = new Map<string, Map<string, ImportControlledDocOrTemplate>>()
private readonly qmsDocsParents = new Map<string, string>()
private readonly qmsDocCodes = new Set<string>()
private readonly qmsTemplatePrefixes = new Set<string>()
private readonly projectTypes = new Map<string, ImportProjectType>()
private readonly issueStatusCache = new Map<string, Ref<IssueStatus>>()
private readonly qmsDocCodeCache = new Set<string>()
private readonly qmsTemplatePrefixCache = new Set<string>()
private readonly errors = new Map<string, ValidationError>()
constructor (
@ -67,6 +73,7 @@ export class ImportWorkspaceBuilder {
async initCache (): Promise<this> {
await this.cacheIssueStatuses()
await this.cacheControlledDocumentCodes()
return this
}
@ -156,13 +163,6 @@ export class ImportWorkspaceBuilder {
throw new Error(`Document space ${spacePath} not found`)
}
if (doc.code !== undefined) {
const duplicateDoc = Array.from(docs.values()).find((existingDoc) => existingDoc.code === doc.code)
if (duplicateDoc !== undefined) {
throw new Error(`Duplicate document code ${doc.code} in space ${spacePath}`)
}
}
this.validateAndAdd(
'controlledDocument',
docPath,
@ -194,13 +194,6 @@ export class ImportWorkspaceBuilder {
throw new Error(`Document space ${spacePath} not found`)
}
if (template.code !== undefined) {
const duplicate = Array.from(qmsDocs.values()).find((existingDoc) => existingDoc.code === template.code)
if (duplicate !== undefined) {
throw new Error(`Duplicate document code ${template.code} in space ${spacePath}`)
}
}
this.validateAndAdd(
'documentTemplate',
templatePath,
@ -304,6 +297,22 @@ export class ImportWorkspaceBuilder {
}
}
async cacheControlledDocumentCodes (): Promise<void> {
const existingDocs = await this.client.findAll(documents.class.Document, {})
for (const doc of existingDocs) {
if (doc.code !== undefined) {
this.qmsDocCodeCache.add(doc.code)
}
}
const existingTemplates = await this.client.findAll(documents.mixin.DocumentTemplate, {})
for (const template of existingTemplates) {
if (template.docPrefix !== undefined) {
this.qmsTemplatePrefixCache.add(template.docPrefix)
}
}
}
private validateAndAdd<T, K>(
type: string,
path: string,
@ -602,7 +611,7 @@ export class ImportWorkspaceBuilder {
issue.subdocs = childIssues
}
private buildControlledDocumentHierarchy (docPath: string, allDocs: Map<string, ImportControlledDoc>): void {
private buildControlledDocumentHierarchy (docPath: string, allDocs: Map<string, ImportControlledDocOrTemplate>): void {
const doc = allDocs.get(docPath)
if (doc === undefined) return
@ -707,8 +716,21 @@ export class ImportWorkspaceBuilder {
errors.push(...this.validateType(doc.class, 'string', 'class'))
errors.push(...this.validateType(doc.template, 'string', 'template'))
errors.push(...this.validateType(doc.state, 'string', 'state'))
if (doc.code !== undefined) {
errors.push(...this.validateType(doc.code, 'string', 'code'))
// Validate if document code exists in the database
if (this.qmsDocCodeCache.has(doc.code)) {
errors.push(`Document with code ${doc.code} already exists in the database`)
}
// Validate if document code is unique among imported documents
if (this.qmsDocCodes.has(doc.code)) {
errors.push(`Duplicate document code ${doc.code} in import`)
} else {
this.qmsDocCodes.add(doc.code)
}
}
// Validate required string fields are defined
@ -754,8 +776,6 @@ export class ImportWorkspaceBuilder {
errors.push('invalid state: ' + doc.state)
}
// todo: validate seqNumber is not duplicated (unique prefix? code?)
return errors
}
@ -767,13 +787,37 @@ export class ImportWorkspaceBuilder {
errors.push(...this.validateType(template.class, 'string', 'class'))
errors.push(...this.validateType(template.docPrefix, 'string', 'docPrefix'))
errors.push(...this.validateType(template.state, 'string', 'state'))
if (template.code !== undefined) {
errors.push(...this.validateType(template.code, 'string', 'code'))
// Validate if template code exists in the database
if (this.qmsDocCodeCache.has(template.code)) {
errors.push(`Template with code ${template.code} already exists in the database`)
}
// Validate if template code is unique among imported templates
if (this.qmsDocCodes.has(template.code)) {
errors.push(`Duplicate template code ${template.code} in import`)
} else {
this.qmsDocCodes.add(template.code)
}
}
// Validate required string fields are defined
if (!this.validateStringDefined(template.title)) errors.push('title is required')
if (!this.validateStringDefined(template.docPrefix)) errors.push('docPrefix is required')
// Validate if template prefix exists in the database
if (this.qmsTemplatePrefixCache.has(template.docPrefix)) {
errors.push(`Template with documents prefix ${template.docPrefix} already exists in the database`)
}
// Validate if template prefix is unique among imported templates
if (this.qmsTemplatePrefixes.has(template.docPrefix)) {
errors.push(`Duplicate template documents prefix ${template.docPrefix} in import`)
} else {
this.qmsTemplatePrefixes.add(template.docPrefix)
}
// Validate numbers are positive
if (!this.validatePossitiveNumber(template.major)) errors.push('invalid value for field "major"')
@ -814,25 +858,6 @@ export class ImportWorkspaceBuilder {
errors.push('invalid state: ' + template.state)
}
// todo: validate seqNumber no duplicated
return errors
}
private validateControlledDocumentSpaces (): void {
// Validate document spaces
for (const [spacePath] of this.qmsSpaces) {
// Validate controlled documents
const docs = this.qmsDocsBySpace.get(spacePath)
if (docs !== undefined) {
// for (const [docPath, doc] of docs) {
for (const docPath of docs.keys()) {
// Check parent document exists
const parentPath = this.documentParents.get(docPath)
if (parentPath !== undefined && !docs.has(parentPath)) {
this.addError(docPath, `Parent document not found: ${parentPath}`)
}
}
}
}
}
}

65
packages/importer/src/importer/importer.ts
View File

@ -179,8 +179,8 @@ export interface ImportDrawing {
contentProvider: () => Promise<string>
}
export type ImportControlledDoc = ImportControlledDocument | ImportControlledDocumentTemplate // todo: rename
export interface ImportOrgSpace extends ImportSpace<ImportControlledDoc> {
export type ImportControlledDocOrTemplate = ImportControlledDocument | ImportControlledDocumentTemplate
export interface ImportOrgSpace extends ImportSpace<ImportControlledDocOrTemplate> {
class: Ref<Class<DocumentSpace>>
qualified?: Ref<Account>
manager?: Ref<Account>
@ -206,7 +206,7 @@ export interface ImportControlledDocumentTemplate extends ImportDoc {
ccReason?: string
ccImpact?: string
ccDescription?: string
subdocs: ImportControlledDoc[]
subdocs: ImportControlledDocOrTemplate[]
}
export interface ImportControlledDocument extends ImportDoc {
@ -218,6 +218,7 @@ export interface ImportControlledDocument extends ImportDoc {
major: number
minor: number
state: DocumentState
category?: Ref<DocumentCategory>
reviewers?: Ref<Employee>[]
approvers?: Ref<Employee>[]
coAuthors?: Ref<Employee>[]
@ -227,7 +228,7 @@ export interface ImportControlledDocument extends ImportDoc {
ccReason?: string
ccImpact?: string
ccDescription?: string
subdocs: ImportControlledDoc[]
subdocs: ImportControlledDocOrTemplate[]
}
export class WorkspaceImporter {
@ -853,7 +854,7 @@ export class WorkspaceImporter {
}
private partitionTemplatesFromDocuments (
doc: ImportControlledDoc,
doc: ImportControlledDocOrTemplate,
documentMap: Map<Ref<ControlledDocument>, ImportControlledDocument>,
templateMap: Map<Ref<ControlledDocument>, ImportControlledDocumentTemplate>
): void {
@ -916,7 +917,7 @@ export class WorkspaceImporter {
this.logger.log('Creating document template: ' + template.title)
const templateId = template.id ?? generateId<ControlledDocument>()
const { seqNumber, code, projectDocumentId } = await createDocumentTemplateMetadata(
const { seqNumber, code, projectDocumentId, success } = await createDocumentTemplateMetadata(
this.client,
documents.class.Document,
spaceId,
@ -930,6 +931,10 @@ export class WorkspaceImporter {
template.metaId
)
if (!success) {
throw new Error('Failed to create document template: ' + template.title)
}
templateMetaMap.set(templateId, { seqNumber, code })
for (const subdoc of template.subdocs) {
@ -966,10 +971,12 @@ export class WorkspaceImporter {
const collabId = makeCollabId(documents.class.Document, template.id, 'content')
const contentId = await this.createCollaborativeContent(template.id, collabId, content, spaceId)
const changeControlId =
template.ccReason !== undefined || template.ccImpact !== undefined || template.ccDescription !== undefined
? await this.createChangeControl(spaceId, template.ccDescription, template.ccReason, template.ccImpact)
: ('' as Ref<ChangeControl>)
const changeControlId = await this.createChangeControl(
spaceId,
template.ccDescription,
template.ccReason,
template.ccImpact
)
const ops = this.client.apply()
const result = await ops.addCollection(
@ -986,19 +993,20 @@ export class WorkspaceImporter {
author: template.author,
owner: template.owner,
abstract: template.abstract,
category: template.category,
reviewers: template.reviewers ?? [],
approvers: template.approvers ?? [],
coAuthors: template.coAuthors ?? [],
code,
seqNumber,
prefix: template.docPrefix, // todo: or TEMPLATE_PREFIX?s
prefix: template.docPrefix,
content: contentId,
changeControl: changeControlId,
commentSequence: 0,
requests: 0,
labels: 0
},
template.id as unknown as Ref<ControlledDocument> // todo: make sure it's not used anywhere as mixin id
template.id as unknown as Ref<ControlledDocument>
)
await ops.createMixin(template.id, documents.class.Document, spaceId, documents.mixin.DocumentTemplate, {
@ -1024,10 +1032,9 @@ export class WorkspaceImporter {
this.logger.log('Creating controlled document: ' + doc.title)
const documentId = doc.id ?? generateId<ControlledDocument>()
// const { seqNumber, prefix, category } = await useDocumentTemplate(this.client, doc.template as unknown as Ref<DocumentTemplate>)
const result = await createControlledDocMetadata(
this.client,
documents.template.ProductChangeControl, // todo: make it dynamic - wtf, commit missed?
documents.template.ProductChangeControl,
documentId,
spaceId,
undefined, // project
@ -1039,6 +1046,10 @@ export class WorkspaceImporter {
doc.metaId
)
if (!result.success) {
throw new Error('Failed to create controlled document: ' + doc.title)
}
// Process subdocs recursively
for (const subdoc of doc.subdocs) {
if (this.isDocumentTemplate(subdoc)) {
@ -1072,18 +1083,22 @@ export class WorkspaceImporter {
const contentId = await this.createCollaborativeContent(document.id, collabId, content, spaceId)
const templateId = document.template
const { seqNumber, prefix, category } = await useDocumentTemplate(
this.client,
templateId as unknown as Ref<DocumentTemplate>
)
const {
seqNumber,
prefix,
category: templateCategory
} = await useDocumentTemplate(this.client, templateId as unknown as Ref<DocumentTemplate>)
const ops = this.client.apply()
const changeControlId =
document.ccReason !== undefined || document.ccImpact !== undefined
? await this.createChangeControl(spaceId, document.ccDescription, document.ccReason, document.ccImpact)
: ('' as Ref<ChangeControl>)
const changeControlId = await this.createChangeControl(
spaceId,
document.ccDescription,
document.ccReason,
document.ccImpact
)
const code = document.code ?? `${prefix}-${seqNumber}`
const result = await ops.addCollection(
documents.class.ControlledDocument,
spaceId,
@ -1102,9 +1117,9 @@ export class WorkspaceImporter {
approvers: document.approvers ?? [],
coAuthors: document.coAuthors ?? [],
changeControl: changeControlId,
code: document.code ?? `${prefix}-${seqNumber}`,
code,
prefix,
category,
category: document.category ?? templateCategory,
seqNumber,
content: contentId,
template: templateId as unknown as Ref<DocumentTemplate>,
@ -1116,7 +1131,7 @@ export class WorkspaceImporter {
await ops.updateDoc(documents.class.DocumentMeta, spaceId, document.metaId, {
documents: 0,
title: `${prefix}-${seqNumber} ${document.title}`
title: `${code} ${document.title}`
})
await ops.commit()

2
plugins/controlled-documents/src/docutils.ts
View File

@ -323,7 +323,7 @@ export async function createDocumentTemplateMetadata (
space,
{
documents: 0,
title: `${TEMPLATE_PREFIX}-${seqNumber} ${specTitle}`
title: `${code} ${specTitle}`
},
metaId
)