From 55543c72d3f7a6cddde8a805da44bf00caf3efa5 Mon Sep 17 00:00:00 2001 From: Anna Khismatullina Date: Tue, 22 Apr 2025 17:40:05 +0700 Subject: [PATCH] Controlled documents import fixes (#8615) Signed-off-by: Anna Khismatullina --- packages/importer/src/huly/cards.ts | 22 ++-- packages/importer/src/huly/huly.ts | 23 ++++- packages/importer/src/huly/registry.ts | 4 +- packages/importer/src/importer/builder.ts | 103 ++++++++++++------- packages/importer/src/importer/importer.ts | 65 +++++++----- plugins/controlled-documents/src/docutils.ts | 2 +- 6 files changed, 138 insertions(+), 81 deletions(-) diff --git a/packages/importer/src/huly/cards.ts b/packages/importer/src/huly/cards.ts index 1ec3dbd43e..d9fac979e0 100644 --- a/packages/importer/src/huly/cards.ts +++ b/packages/importer/src/huly/cards.ts @@ -81,20 +81,22 @@ export class CardsProcessor { await this.processMetadata(directoryPath, result, topLevelTypes) const typesRefs = topLevelTypes.map((type) => type.props._id) as Ref[] - const updateDefaultSpace: UnifiedUpdate = { - _class: card.class.CardSpace, - _id: 'card:space:Default' as Ref, - space: core.space.Model, - props: { - $push: { - types: { - $each: [...new Set(typesRefs)], - $position: 0 + if (typesRefs.length > 0) { + const updateDefaultSpace: UnifiedUpdate = { + _class: card.class.CardSpace, + _id: 'card:space:Default' as Ref, + space: core.space.Model, + props: { + $push: { + types: { + $each: [...new Set(typesRefs)], + $position: 0 + } } } } + result.updates.set('card:space:Default', [updateDefaultSpace]) } - result.updates.set('card:space:Default', [updateDefaultSpace]) await this.processSystemTypeCards(directoryPath, result, new Map(), new Map()) await this.processCards(directoryPath, result, new Map(), new Map()) diff --git a/packages/importer/src/huly/huly.ts b/packages/importer/src/huly/huly.ts index 2b7b96ec65..ede29fcb89 100644 --- a/packages/importer/src/huly/huly.ts +++ b/packages/importer/src/huly/huly.ts @@ -126,6 +126,7 @@ export interface HulyControlledDocumentHeader { template: string author: string owner: string + category?: string abstract?: string reviewers?: string[] approvers?: string[] @@ -320,6 +321,7 @@ export class HulyFormatImporter { private personsByName = new Map>() private accountsByEmail = new Map>() private employeesByName = new Map>() + private controlledDocumentCategories = new Map>() private readonly fileMetaByPath = new Map() @@ -338,6 +340,7 @@ export class HulyFormatImporter { await this.cachePersonsByNames() await this.cacheAccountsByEmails() await this.cacheEmployeesByName() + await this.cacheControlledDocumentCategories() } async importFolder (folderPath: string): Promise { @@ -674,8 +677,8 @@ export class HulyFormatImporter { throw new Error(`Unknown document class ${docHeader.class} in ${docFile}`) } - const documentMetaId = this.metadataRegistry.getRef(docPath) as Ref - this.metadataRegistry.setRefMetadata(docPath, documents.class.DocumentMeta, docHeader.title) + const documentMetaId = generateId() + this.metadataRegistry.setRefMetadata(docPath, documents.class.DocumentMeta, docHeader.title, documentMetaId) if (docHeader.class === documents.class.ControlledDocument) { const doc = await this.processControlledDocument( @@ -814,7 +817,7 @@ export class HulyFormatImporter { } const templateId = this.metadataRegistry.getRef(templatePath) as Ref - + const category = header.category !== undefined ? this.controlledDocumentCategories.get(header.category) : undefined return { id, metaId, @@ -825,6 +828,7 @@ export class HulyFormatImporter { major: 0, minor: 1, state: DocumentState.Draft, + category, author, owner, abstract: header.abstract, @@ -852,6 +856,7 @@ export class HulyFormatImporter { } const codeMatch = path.basename(docPath).match(/^\[([^\]]+)\]/) + const category = header.category !== undefined ? this.controlledDocumentCategories.get(header.category) : undefined return { id, metaId, @@ -862,7 +867,7 @@ export class HulyFormatImporter { major: 0, minor: 1, state: DocumentState.Draft, - category: header.category as Ref, + category, author, owner, abstract: header.abstract, @@ -913,6 +918,16 @@ export class HulyFormatImporter { }, new Map()) } + private async cacheControlledDocumentCategories (): Promise { + this.controlledDocumentCategories = (await this.client.findAll(documents.class.DocumentCategory, {})).reduce( + (refByCode, category) => { + refByCode.set(category.code, category._id) + return refByCode + }, + new Map() + ) + } + private async collectFileMetadata (folderPath: string): Promise { const processDir = async (dir: string): Promise => { const entries = fs.readdirSync(dir, { withFileTypes: true }) diff --git a/packages/importer/src/huly/registry.ts b/packages/importer/src/huly/registry.ts index 484f6e1496..2c9443176d 100644 --- a/packages/importer/src/huly/registry.ts +++ b/packages/importer/src/huly/registry.ts @@ -106,8 +106,8 @@ export class MetadataRegistry { this.pathToTagMetadata.set(tagPath, metadata) } - public setRefMetadata (path: string, _class: string, title: string): void { - const ref = this.getRef(path) + public setRefMetadata (path: string, _class: string, title: string, id?: Ref): void { + const ref = id ?? this.getRef(path) this.pathToMentionMetadata.set(path, { id: ref, class: _class, diff --git a/packages/importer/src/importer/builder.ts b/packages/importer/src/importer/builder.ts index 3442468f51..ed649af2c7 100644 --- a/packages/importer/src/importer/builder.ts +++ b/packages/importer/src/importer/builder.ts @@ -20,7 +20,7 @@ import { ImportControlledDocument, ImportControlledDocumentTemplate, ImportOrgSpace, - type ImportControlledDoc, + type ImportControlledDocOrTemplate, type ImportDocument, type ImportIssue, type ImportProject, @@ -43,6 +43,8 @@ const MAX_PROJECT_IDENTIFIER_LENGTH = 5 const PROJECT_IDENTIFIER_REGEX = /^[a-zA-Z_][a-zA-Z0-9_]*$/ export class ImportWorkspaceBuilder { + private readonly projectTypes = new Map() + private readonly projects = new Map() private readonly issuesByProject = new Map>() private readonly issueParents = new Map() @@ -53,11 +55,15 @@ export class ImportWorkspaceBuilder { private readonly qmsSpaces = new Map() private readonly qmsTemplates = new Map, string>() - private readonly qmsDocsBySpace = new Map>() + private readonly qmsDocsBySpace = new Map>() private readonly qmsDocsParents = new Map() + private readonly qmsDocCodes = new Set() + private readonly qmsTemplatePrefixes = new Set() - private readonly projectTypes = new Map() private readonly issueStatusCache = new Map>() + private readonly qmsDocCodeCache = new Set() + private readonly qmsTemplatePrefixCache = new Set() + private readonly errors = new Map() constructor ( @@ -67,6 +73,7 @@ export class ImportWorkspaceBuilder { async initCache (): Promise { await this.cacheIssueStatuses() + await this.cacheControlledDocumentCodes() return this } @@ -156,13 +163,6 @@ export class ImportWorkspaceBuilder { throw new Error(`Document space ${spacePath} not found`) } - if (doc.code !== undefined) { - const duplicateDoc = Array.from(docs.values()).find((existingDoc) => existingDoc.code === doc.code) - if (duplicateDoc !== undefined) { - throw new Error(`Duplicate document code ${doc.code} in space ${spacePath}`) - } - } - this.validateAndAdd( 'controlledDocument', docPath, @@ -194,13 +194,6 @@ export class ImportWorkspaceBuilder { throw new Error(`Document space ${spacePath} not found`) } - if (template.code !== undefined) { - const duplicate = Array.from(qmsDocs.values()).find((existingDoc) => existingDoc.code === template.code) - if (duplicate !== undefined) { - throw new Error(`Duplicate document code ${template.code} in space ${spacePath}`) - } - } - this.validateAndAdd( 'documentTemplate', templatePath, @@ -304,6 +297,22 @@ export class ImportWorkspaceBuilder { } } + async cacheControlledDocumentCodes (): Promise { + const existingDocs = await this.client.findAll(documents.class.Document, {}) + for (const doc of existingDocs) { + if (doc.code !== undefined) { + this.qmsDocCodeCache.add(doc.code) + } + } + + const existingTemplates = await this.client.findAll(documents.mixin.DocumentTemplate, {}) + for (const template of existingTemplates) { + if (template.docPrefix !== undefined) { + this.qmsTemplatePrefixCache.add(template.docPrefix) + } + } + } + private validateAndAdd( type: string, path: string, @@ -602,7 +611,7 @@ export class ImportWorkspaceBuilder { issue.subdocs = childIssues } - private buildControlledDocumentHierarchy (docPath: string, allDocs: Map): void { + private buildControlledDocumentHierarchy (docPath: string, allDocs: Map): void { const doc = allDocs.get(docPath) if (doc === undefined) return @@ -707,8 +716,21 @@ export class ImportWorkspaceBuilder { errors.push(...this.validateType(doc.class, 'string', 'class')) errors.push(...this.validateType(doc.template, 'string', 'template')) errors.push(...this.validateType(doc.state, 'string', 'state')) + if (doc.code !== undefined) { errors.push(...this.validateType(doc.code, 'string', 'code')) + + // Validate if document code exists in the database + if (this.qmsDocCodeCache.has(doc.code)) { + errors.push(`Document with code ${doc.code} already exists in the database`) + } + + // Validate if document code is unique among imported documents + if (this.qmsDocCodes.has(doc.code)) { + errors.push(`Duplicate document code ${doc.code} in import`) + } else { + this.qmsDocCodes.add(doc.code) + } } // Validate required string fields are defined @@ -754,8 +776,6 @@ export class ImportWorkspaceBuilder { errors.push('invalid state: ' + doc.state) } - // todo: validate seqNumber is not duplicated (unique prefix? code?) - return errors } @@ -767,13 +787,37 @@ export class ImportWorkspaceBuilder { errors.push(...this.validateType(template.class, 'string', 'class')) errors.push(...this.validateType(template.docPrefix, 'string', 'docPrefix')) errors.push(...this.validateType(template.state, 'string', 'state')) + if (template.code !== undefined) { errors.push(...this.validateType(template.code, 'string', 'code')) + + // Validate if template code exists in the database + if (this.qmsDocCodeCache.has(template.code)) { + errors.push(`Template with code ${template.code} already exists in the database`) + } + + // Validate if template code is unique among imported templates + if (this.qmsDocCodes.has(template.code)) { + errors.push(`Duplicate template code ${template.code} in import`) + } else { + this.qmsDocCodes.add(template.code) + } } // Validate required string fields are defined if (!this.validateStringDefined(template.title)) errors.push('title is required') if (!this.validateStringDefined(template.docPrefix)) errors.push('docPrefix is required') + // Validate if template prefix exists in the database + if (this.qmsTemplatePrefixCache.has(template.docPrefix)) { + errors.push(`Template with documents prefix ${template.docPrefix} already exists in the database`) + } + + // Validate if template prefix is unique among imported templates + if (this.qmsTemplatePrefixes.has(template.docPrefix)) { + errors.push(`Duplicate template documents prefix ${template.docPrefix} in import`) + } else { + this.qmsTemplatePrefixes.add(template.docPrefix) + } // Validate numbers are positive if (!this.validatePossitiveNumber(template.major)) errors.push('invalid value for field "major"') @@ -814,25 +858,6 @@ export class ImportWorkspaceBuilder { errors.push('invalid state: ' + template.state) } - // todo: validate seqNumber no duplicated return errors } - - private validateControlledDocumentSpaces (): void { - // Validate document spaces - for (const [spacePath] of this.qmsSpaces) { - // Validate controlled documents - const docs = this.qmsDocsBySpace.get(spacePath) - if (docs !== undefined) { - // for (const [docPath, doc] of docs) { - for (const docPath of docs.keys()) { - // Check parent document exists - const parentPath = this.documentParents.get(docPath) - if (parentPath !== undefined && !docs.has(parentPath)) { - this.addError(docPath, `Parent document not found: ${parentPath}`) - } - } - } - } - } } diff --git a/packages/importer/src/importer/importer.ts b/packages/importer/src/importer/importer.ts index 3c3f0087b8..d0516553f0 100644 --- a/packages/importer/src/importer/importer.ts +++ b/packages/importer/src/importer/importer.ts @@ -179,8 +179,8 @@ export interface ImportDrawing { contentProvider: () => Promise } -export type ImportControlledDoc = ImportControlledDocument | ImportControlledDocumentTemplate // todo: rename -export interface ImportOrgSpace extends ImportSpace { +export type ImportControlledDocOrTemplate = ImportControlledDocument | ImportControlledDocumentTemplate +export interface ImportOrgSpace extends ImportSpace { class: Ref> qualified?: Ref manager?: Ref @@ -206,7 +206,7 @@ export interface ImportControlledDocumentTemplate extends ImportDoc { ccReason?: string ccImpact?: string ccDescription?: string - subdocs: ImportControlledDoc[] + subdocs: ImportControlledDocOrTemplate[] } export interface ImportControlledDocument extends ImportDoc { @@ -218,6 +218,7 @@ export interface ImportControlledDocument extends ImportDoc { major: number minor: number state: DocumentState + category?: Ref reviewers?: Ref[] approvers?: Ref[] coAuthors?: Ref[] @@ -227,7 +228,7 @@ export interface ImportControlledDocument extends ImportDoc { ccReason?: string ccImpact?: string ccDescription?: string - subdocs: ImportControlledDoc[] + subdocs: ImportControlledDocOrTemplate[] } export class WorkspaceImporter { @@ -853,7 +854,7 @@ export class WorkspaceImporter { } private partitionTemplatesFromDocuments ( - doc: ImportControlledDoc, + doc: ImportControlledDocOrTemplate, documentMap: Map, ImportControlledDocument>, templateMap: Map, ImportControlledDocumentTemplate> ): void { @@ -916,7 +917,7 @@ export class WorkspaceImporter { this.logger.log('Creating document template: ' + template.title) const templateId = template.id ?? generateId() - const { seqNumber, code, projectDocumentId } = await createDocumentTemplateMetadata( + const { seqNumber, code, projectDocumentId, success } = await createDocumentTemplateMetadata( this.client, documents.class.Document, spaceId, @@ -930,6 +931,10 @@ export class WorkspaceImporter { template.metaId ) + if (!success) { + throw new Error('Failed to create document template: ' + template.title) + } + templateMetaMap.set(templateId, { seqNumber, code }) for (const subdoc of template.subdocs) { @@ -966,10 +971,12 @@ export class WorkspaceImporter { const collabId = makeCollabId(documents.class.Document, template.id, 'content') const contentId = await this.createCollaborativeContent(template.id, collabId, content, spaceId) - const changeControlId = - template.ccReason !== undefined || template.ccImpact !== undefined || template.ccDescription !== undefined - ? await this.createChangeControl(spaceId, template.ccDescription, template.ccReason, template.ccImpact) - : ('' as Ref) + const changeControlId = await this.createChangeControl( + spaceId, + template.ccDescription, + template.ccReason, + template.ccImpact + ) const ops = this.client.apply() const result = await ops.addCollection( @@ -986,19 +993,20 @@ export class WorkspaceImporter { author: template.author, owner: template.owner, abstract: template.abstract, + category: template.category, reviewers: template.reviewers ?? [], approvers: template.approvers ?? [], coAuthors: template.coAuthors ?? [], code, seqNumber, - prefix: template.docPrefix, // todo: or TEMPLATE_PREFIX?s + prefix: template.docPrefix, content: contentId, changeControl: changeControlId, commentSequence: 0, requests: 0, labels: 0 }, - template.id as unknown as Ref // todo: make sure it's not used anywhere as mixin id + template.id as unknown as Ref ) await ops.createMixin(template.id, documents.class.Document, spaceId, documents.mixin.DocumentTemplate, { @@ -1024,10 +1032,9 @@ export class WorkspaceImporter { this.logger.log('Creating controlled document: ' + doc.title) const documentId = doc.id ?? generateId() - // const { seqNumber, prefix, category } = await useDocumentTemplate(this.client, doc.template as unknown as Ref) const result = await createControlledDocMetadata( this.client, - documents.template.ProductChangeControl, // todo: make it dynamic - wtf, commit missed? + documents.template.ProductChangeControl, documentId, spaceId, undefined, // project @@ -1039,6 +1046,10 @@ export class WorkspaceImporter { doc.metaId ) + if (!result.success) { + throw new Error('Failed to create controlled document: ' + doc.title) + } + // Process subdocs recursively for (const subdoc of doc.subdocs) { if (this.isDocumentTemplate(subdoc)) { @@ -1072,18 +1083,22 @@ export class WorkspaceImporter { const contentId = await this.createCollaborativeContent(document.id, collabId, content, spaceId) const templateId = document.template - const { seqNumber, prefix, category } = await useDocumentTemplate( - this.client, - templateId as unknown as Ref - ) + const { + seqNumber, + prefix, + category: templateCategory + } = await useDocumentTemplate(this.client, templateId as unknown as Ref) const ops = this.client.apply() - const changeControlId = - document.ccReason !== undefined || document.ccImpact !== undefined - ? await this.createChangeControl(spaceId, document.ccDescription, document.ccReason, document.ccImpact) - : ('' as Ref) + const changeControlId = await this.createChangeControl( + spaceId, + document.ccDescription, + document.ccReason, + document.ccImpact + ) + const code = document.code ?? `${prefix}-${seqNumber}` const result = await ops.addCollection( documents.class.ControlledDocument, spaceId, @@ -1102,9 +1117,9 @@ export class WorkspaceImporter { approvers: document.approvers ?? [], coAuthors: document.coAuthors ?? [], changeControl: changeControlId, - code: document.code ?? `${prefix}-${seqNumber}`, + code, prefix, - category, + category: document.category ?? templateCategory, seqNumber, content: contentId, template: templateId as unknown as Ref, @@ -1116,7 +1131,7 @@ export class WorkspaceImporter { await ops.updateDoc(documents.class.DocumentMeta, spaceId, document.metaId, { documents: 0, - title: `${prefix}-${seqNumber} ${document.title}` + title: `${code} ${document.title}` }) await ops.commit() diff --git a/plugins/controlled-documents/src/docutils.ts b/plugins/controlled-documents/src/docutils.ts index 43224cedf1..1e980e094c 100644 --- a/plugins/controlled-documents/src/docutils.ts +++ b/plugins/controlled-documents/src/docutils.ts @@ -323,7 +323,7 @@ export async function createDocumentTemplateMetadata ( space, { documents: 0, - title: `${TEMPLATE_PREFIX}-${seqNumber} ${specTitle}` + title: `${code} ${specTitle}` }, metaId )