diff --git a/.vscode/launch.json b/.vscode/launch.json index 63d193057f..dc42481c62 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -51,7 +51,6 @@ "name": "Debug tool", "type": "node", "request": "launch", - // "args": ["src/index.ts", "import-xml", "ws1", "/Users/haiodo/Develop/private/hardware/suho/Кандидаты/Кандидаты.xml"], "args": ["src/index.ts", "restore-workspace", "ws1", "../../temp/ws1/"], "env": { "MINIO_ACCESS_KEY":"minioadmin", diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index b5f13d963c..ed8d25f1a1 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -116,6 +116,7 @@ specifiers: '@types/pdfkit': ~0.12.3 '@types/toposort': ^2.0.3 '@types/uuid': ^8.3.1 + '@types/xml2js': ~0.4.9 '@typescript-eslint/eslint-plugin': ^5.4.0 autoprefixer: ^10.2.6 commander: ^8.1.0 @@ -144,6 +145,7 @@ specifiers: koa-bodyparser: ^4.3.0 koa-router: ^10.1.1 lexorank: ~1.0.4 + mime-types: ~2.1.34 mini-css-extract-plugin: ^2.2.0 minio: ^7.0.19 node-html-parser: ^4.1.3 @@ -168,6 +170,7 @@ specifiers: webpack-bundle-analyzer: ^4.4.1 webpack-cli: ^4.6.0 webpack-dev-server: ^3.11.2 + xml2js: ~0.4.23 dependencies: '@elastic/elasticsearch': 7.16.0 @@ -285,6 +288,7 @@ dependencies: '@types/pdfkit': 0.12.3 '@types/toposort': 2.0.3 '@types/uuid': 8.3.3 + '@types/xml2js': 0.4.9 '@typescript-eslint/eslint-plugin': 5.7.0_eslint@7.32.0+typescript@4.5.4 autoprefixer: 10.4.0_postcss@8.4.5 commander: 8.3.0 @@ -313,6 +317,7 @@ dependencies: koa-bodyparser: 4.3.0 koa-router: 10.1.1 lexorank: 1.0.4 + mime-types: 2.1.34 mini-css-extract-plugin: 2.4.5_webpack@5.65.0 minio: 7.0.25 node-html-parser: 4.1.5 @@ -337,6 +342,7 @@ dependencies: webpack-bundle-analyzer: 4.5.0 webpack-cli: 4.9.1_1cfd1380a4e9b8401fb780accef05e9c webpack-dev-server: 3.11.3_webpack-cli@4.9.1+webpack@5.65.0 + xml2js: 0.4.23 packages: @@ -1713,6 +1719,10 @@ packages: '@types/koa': 2.13.4 dev: false + /@types/mime-types/2.1.1: + resolution: {integrity: sha512-vXOTGVSLR2jMw440moWTC7H19iUyLtP3Z1YTj7cSsubOICinjMxFeb/V57v9QdyyPGbbWolUFSSmSiRSn94tFw==} + dev: false + /@types/mime/1.3.2: resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==} dev: false @@ -1902,6 +1912,12 @@ packages: '@types/node': 17.0.0 dev: false + /@types/xml2js/0.4.9: + resolution: {integrity: sha512-CHiCKIihl1pychwR2RNX5mAYmJDACgFVCMT5OArMaO3erzwXVcBqPcusr+Vl8yeeXukxZqtF8mZioqX+mpjjdw==} + dependencies: + '@types/node': 17.0.0 + dev: false + /@types/yargs-parser/20.2.1: resolution: {integrity: sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw==} dev: false @@ -11019,7 +11035,7 @@ packages: dev: false file:projects/dev-client-resources.tgz: - resolution: {integrity: sha512-d1gACtHmA2R8CbItLx5oJYfVQV7FvM4tQthnwsDeoIM9OUIlXaRVFhd/KlJf1MkqLIsOA4O+Gjvhih/+EtediA==, tarball: file:projects/dev-client-resources.tgz} + resolution: {integrity: sha512-hcwOdV5TgwA6yTNeZDtMwAaGEOYFO5VzqWTXk+sK+SPSiEN2yy6rb2Do1cfgL0wvSXLH/F2VT+nr1WAlI909lQ==, tarball: file:projects/dev-client-resources.tgz} name: '@rush-temp/dev-client-resources' version: 0.0.0 dependencies: @@ -12511,16 +12527,18 @@ packages: dev: false file:projects/tool.tgz: - resolution: {integrity: sha512-FQTXToKTNRIcen67zjbh/ZeibEPSskFtKhFZgTtUgB0TXhzJ0V+ZRcpnsXK+nND2DBBRYhtylQJmqWsoASRtFg==, tarball: file:projects/tool.tgz} + resolution: {integrity: sha512-kxAczrVzjVh47GTnScTkjjXIpSAkovYmCIxJSznbPqT4XOdtQ0F8eadl3h2mhqjKzgE7VoFOdTB5JJ3oy4+TUg==, tarball: file:projects/tool.tgz} name: '@rush-temp/tool' version: 0.0.0 dependencies: '@elastic/elasticsearch': 7.16.0 '@rushstack/heft': 0.41.8 '@types/heft-jest': 1.0.2 + '@types/mime-types': 2.1.1 '@types/minio': 7.0.11 '@types/node': 16.11.14 '@types/ws': 8.2.2 + '@types/xml2js': 0.4.9 '@typescript-eslint/eslint-plugin': 5.7.0_c25e8c1f4f4f7aaed27aa6f9ce042237 '@typescript-eslint/parser': 5.7.0_eslint@7.32.0+typescript@4.5.4 commander: 8.3.0 @@ -12533,12 +12551,14 @@ packages: eslint-plugin-promise: 5.2.0_eslint@7.32.0 fast-equals: 2.0.4 jwt-simple: 0.5.6 + mime-types: 2.1.34 minio: 7.0.25 mongodb: 4.2.2 prettier: 2.5.1 ts-node: 10.4.0_5d12c2add188ff0e728b4ade3dacd39b typescript: 4.5.4 ws: 8.3.0 + xml2js: 0.4.23 transitivePeerDependencies: - '@swc/core' - '@swc/wasm' diff --git a/dev/tool/package.json b/dev/tool/package.json index aff66608d4..8d9f5c9b17 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -34,7 +34,9 @@ "prettier": "^2.4.1", "@rushstack/heft": "^0.41.1", "typescript": "^4.3.5", - "@types/ws": "^8.2.1" + "@types/ws": "^8.2.1", + "@types/xml2js": "~0.4.9", + "@types/mime-types": "~2.1.1" }, "dependencies": { "mongodb": "^4.1.1", @@ -62,6 +64,13 @@ "@anticrm/server-chunter": "~0.6.1", "@anticrm/server-chunter-resources": "~0.6.0", "@anticrm/server-recruit": "~0.6.0", - "@anticrm/server-recruit-resources": "~0.6.0" + "@anticrm/server-recruit-resources": "~0.6.0", + "xml2js": "~0.4.23", + "@anticrm/model-recruit": "~0.6.0", + "@anticrm/recruit": "~0.6.2", + "@anticrm/task": "~0.6.0", + "@anticrm/chunter": "~0.6.1", + "mime-types": "~2.1.34", + "@anticrm/attachment": "~0.6.1" } } diff --git a/dev/tool/src/elastic.ts b/dev/tool/src/elastic.ts index d529695c9d..81e3ec1ad2 100644 --- a/dev/tool/src/elastic.ts +++ b/dev/tool/src/elastic.ts @@ -34,6 +34,7 @@ import core, { } from '@anticrm/core' import { createElasticAdapter } from '@anticrm/elastic' import { DOMAIN_ATTACHMENT } from '@anticrm/model-attachment' +import { Attachment } from '@anticrm/attachment' import { createMongoAdapter, createMongoTxAdapter } from '@anticrm/mongo' import { addLocation } from '@anticrm/platform' import { serverChunterId } from '@anticrm/server-chunter' @@ -91,29 +92,94 @@ async function dropElastic (elasticUrl: string, dbName: string): Promise { await client.close() } +export class ElasticTool { + mongoClient: MongoClient + elastic!: FullTextAdapter & {close: () => Promise} + storage!: ServerStorage + db!: Db + constructor (readonly mongoUrl: string, readonly dbName: string, readonly minio: Client, readonly elasticUrl: string) { + addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources')) + addLocation(serverRecruitId, () => import('@anticrm/server-recruit-resources')) + this.mongoClient = new MongoClient(mongoUrl) + } + + async connect (): Promise<() => Promise> { + await this.mongoClient.connect() + + this.db = this.mongoClient.db(this.dbName) + this.elastic = await createElasticAdapter(this.elasticUrl, this.dbName) + this.storage = await createStorage(this.mongoUrl, this.elasticUrl, this.dbName) + + return async () => { + await this.mongoClient.close() + await this.elastic.close() + } + } + + async indexAttachment ( + name: string + ): Promise { + const doc: Attachment | null = await this.db.collection(DOMAIN_ATTACHMENT).findOne({ file: name }) + if (doc == null) return + + const buffer = await this.readMinioObject(name) + await this.indexAttachmentDoc(doc, buffer) + } + + async indexAttachmentDoc (doc: Attachment, buffer: Buffer): Promise { + const id: Ref = (generateId() + '/attachments/') as Ref + + const indexedDoc: IndexedDoc = { + id: id, + _class: doc._class, + space: doc.space, + modifiedOn: doc.modifiedOn, + modifiedBy: 'core:account:System' as Ref, + attachedTo: doc.attachedTo, + data: buffer.toString('base64') + } + + await this.elastic.index(indexedDoc) + } + + private async readMinioObject (name: string): Promise { + const data = await this.minio.getObject(this.dbName, name) + const chunks: Buffer[] = [] + await new Promise((resolve) => { + data.on('readable', () => { + let chunk + while ((chunk = data.read()) !== null) { + const b = chunk as Buffer + chunks.push(b) + } + }) + + data.on('end', () => { + resolve() + }) + }) + return Buffer.concat(chunks) + } +} + async function restoreElastic (mongoUrl: string, dbName: string, minio: Client, elasticUrl: string): Promise { - addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources')) - addLocation(serverRecruitId, () => import('@anticrm/server-recruit-resources')) - const mongoClient = new MongoClient(mongoUrl) + const tool = new ElasticTool(mongoUrl, dbName, minio, elasticUrl) + const done = await tool.connect() try { - await mongoClient.connect() - const db = mongoClient.db(dbName) - const elastic = await createElasticAdapter(elasticUrl, dbName) - const storage = await createStorage(mongoUrl, elasticUrl, dbName) - const txes = (await db.collection(DOMAIN_TX).find().sort({ _id: 1 }).toArray()) + const txes = (await tool.db.collection(DOMAIN_TX).find().sort({ _id: 1 }).toArray()) const data = txes.filter((tx) => tx.objectSpace !== core.space.Model) const metricsCtx = new MeasureMetricsContext('elastic', {}) for (const tx of data) { - await storage.tx(metricsCtx, tx) + await tool.storage.tx(metricsCtx, tx) } if (await minio.bucketExists(dbName)) { const minioObjects = await listMinioObjects(minio, dbName) for (const d of minioObjects) { - await indexAttachment(elastic, minio, db, dbName, d.name) + await tool.indexAttachment(d.name) } } } finally { - await mongoClient.close() + await done() } } @@ -142,49 +208,6 @@ async function createStorage (mongoUrl: string, elasticUrl: string, workspace: s return await createServerStorage(conf) } -async function indexAttachment ( - elastic: FullTextAdapter, - minio: Client, - db: Db, - dbName: string, - name: string -): Promise { - const doc = await db.collection(DOMAIN_ATTACHMENT).findOne({ - file: name - }) - if (doc == null) return - - const data = await minio.getObject(dbName, name) - const chunks: Buffer[] = [] - await new Promise((resolve) => { - data.on('readable', () => { - let chunk - while ((chunk = data.read()) !== null) { - const b = chunk as Buffer - chunks.push(b) - } - }) - - data.on('end', () => { - resolve() - }) - }) - - const id: Ref = (generateId() + '/attachments/') as Ref - - const indexedDoc: IndexedDoc = { - id: id, - _class: doc._class, - space: doc.space, - modifiedOn: doc.modifiedOn, - modifiedBy: 'core:account:System' as Ref, - attachedTo: doc.attachedTo, - data: Buffer.concat(chunks).toString('base64') - } - - await elastic.index(indexedDoc) -} - async function createMongoReadOnlyAdapter ( hierarchy: Hierarchy, url: string, diff --git a/dev/tool/src/importer.ts b/dev/tool/src/importer.ts new file mode 100644 index 0000000000..98e4ebe0fc --- /dev/null +++ b/dev/tool/src/importer.ts @@ -0,0 +1,440 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// Copyright © 2021 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import attachment, { Attachment } from '@anticrm/attachment' +import chunter, { Comment } from '@anticrm/chunter' +import contact, { ChannelProvider } from '@anticrm/contact' +import core, { AttachedData, AttachedDoc, Class, Data, Doc, DocumentUpdate, Ref, SortingOrder, Space, TxOperations, TxResult } from '@anticrm/core' +import recruit from '@anticrm/model-recruit' +import { Applicant, Candidate, Vacancy } from '@anticrm/recruit' +import task, { calcRank, DoneState, genRanks, Kanban, State } from '@anticrm/task' +import { deepEqual } from 'fast-equals' +import { existsSync } from 'fs' +import { readdir, readFile, stat } from 'fs/promises' +import mime from 'mime-types' +import { Client } from 'minio' +import { dirname, join } from 'path' +import { parseStringPromise } from 'xml2js' +import { connect } from './connect' +import { ElasticTool } from './elastic' + +const _ = { + candidates: 'Кандидаты', + candidate: 'Кандидат', + + delete: 'ПометкаУдаления', // - + + id: 'Код', // + + fullName: 'Наименование', // + + recruiter: 'Рекрутер', + birthdate: 'ДатаРождения', + phone: 'НомерТелефона', // + + email: 'ЭлектроннаяПочта', // + + date: 'Дата', + + notifyStatus: 'УведомлятьОСменеСтатуса', + + requiredPosition: 'ЖелаемаяДолжность', // + + + vacancyKind: 'ВидВакансии', // + + + city: 'Проживание', // + + comment: 'Комментарий', // + + + socialLink: 'СсылкиНаСоцСети', // + + socialLink2: 'СсылкиНаСоцСети1', // + + socialLink3: 'СсылкиНаСоцСети2', // + + otherContact: 'Другие_КонтактыMyMail', + socialChannel: 'КаналСвязи', // + + + area: 'Отрасль', // + + area2: 'Отрасль1', // + + area3: 'Отрасль2', // + + status: 'Статус', + socialContacted: 'Откликнулся', // + + framework: 'ДополнительныйФреймворкТехнологияПлатформаЛибоВторойЯП' // + +} + +function get (data: any, key: string): string | undefined { + const v = data[key] + if (Array.isArray(v) && v.length === 1) { + return v[0] + } + return v +} + +export async function importXml ( + transactorUrl: string, + dbName: string, + minio: Client, + xmlFile: string, + mongoUrl: string, + elasticUrl: string +): Promise { + const connection = await connect(transactorUrl, dbName) + + const tool = new ElasticTool(mongoUrl, dbName, minio, elasticUrl) + const done = await tool.connect() + try { + console.log('loading xml document...') + + const xmlData = await readFile(xmlFile, 'utf-8') + const data = await parseStringPromise(xmlData) + + const root = dirname(xmlFile) + + const candidates = data[_.candidates][_.candidate] + console.log('Found candidates:', candidates.length) + + // const attributes = new Set() + const client = new TxOperations(connection, core.account.System) + + const statuses = candidates.map((c: any) => get(c, _.status)).filter((c: any) => c !== undefined) + .filter(onlyUniq) + + console.log(statuses) + + const withStatus: {candidateId: Ref, status: string}[] = [] + let pos = 0 + const len = candidates.length as number + for (const c of candidates) { + pos++ + const _id = get(c, _.id) + const _name = get(c, _.fullName) + if (_name === undefined || _id === undefined) { + console.log('error procesing', JSON.stringify(c, undefined, 2)) + return + } + const candId = `tool-import-${_id}` as Ref + const _status = get(c, _.status) + if (_status !== undefined) { + withStatus.push({ candidateId: candId, status: _status }) + } + + await createCandidate(_name, pos, len, c, client, candId) + + // Check and add attachments. + const candidateRoot = join(root, _id) + + if (existsSync(candidateRoot)) { + const files = await readdir(candidateRoot) + for (const f of files) { + const attachId = (candId + f.toLowerCase()) as Ref + const type = mime.contentType(f) + if (typeof type === 'string') { + const fileName = join(candidateRoot, f) + const data = await readFile(fileName) + try { + await minio.statObject(dbName, attachId) + } catch (err: any) { + // No object, put new one. + await minio.putObject(dbName, attachId, data, data.length, { + 'Content-Type': type + }) + } + + const stats = await stat(fileName) + + const attachedDoc = await findOrUpdateAttached(client, recruit.space.CandidatesPublic, attachment.class.Attachment, attachId, { + name: f, + file: attachId, + type: type, + size: stats.size, + lastModified: stats.mtime.getTime() + }, { + attachedTo: candId, + attachedClass: recruit.class.Candidate, + collection: 'attachments' + }) + + await tool.indexAttachmentDoc(attachedDoc, data) + } + } + } + } + + // Create/Update Vacancy and Applicants. + + if (withStatus.length > 0) { + const { states, vacancyId } = await createUpdateVacancy(client, statuses) + + // Applicant num sequence. + const sequence = await client.findOne(task.class.Sequence, { attachedTo: recruit.class.Applicant }) + if (sequence === undefined) { + throw new Error('sequence object not found') + } + + const rankGen = genRanks(withStatus.length) + const firstState = Array.from(states.values())[0] + for (const { candidateId, status } of withStatus) { + const incResult = await client.updateDoc( + task.class.Sequence, + task.space.Sequence, + sequence._id, + { + $inc: { sequence: 1 } + }, + true + ) + + const rank = rankGen.next().value + const state = states.get(status) ?? firstState + + if (rank === undefined) { + throw Error('Failed to generate rank') + } + + const lastOne = await client.findOne( + recruit.class.Applicant, + { state }, + { sort: { rank: SortingOrder.Descending } } + ) + await createApplicant(vacancyId, candidateId, incResult, state, lastOne, client) + } + } + } catch (err: any) { + console.log(err) + } finally { + await done() + console.log('manual closing connection') + await connection.close() + } +} +const onlyUniq = (value: any, index: number, self: any[]): boolean => self.indexOf(value) === index + +async function createApplicant (vacancyId: Ref, candidateId: Ref, incResult: TxResult, state: Ref, lastOne: Applicant | undefined, client: TxOperations): Promise { + const applicantId = `vacancy-${vacancyId}-${candidateId}` as Ref + + const applicant: AttachedData = { + number: (incResult as any).object.sequence, + assignee: null, + state, + doneState: null, + rank: calcRank(lastOne, undefined) + } + + // Update or create candidate + await findOrUpdateAttached(client, vacancyId, recruit.class.Applicant, applicantId, applicant, { attachedTo: candidateId, attachedClass: recruit.class.Candidate, collection: 'applications' }) +} + +async function createUpdateVacancy (client: TxOperations, statuses: any): Promise<{states: Map>, vacancyId: Ref}> { + const vacancy: Data = { + name: 'Imported Vacancy', + description: '', + fullDescription: '', + location: '', + company: '', + members: [], + archived: false, + private: false + } + const vacancyId = 'imported-vacancy' as Ref + + console.log('Creating vacancy', vacancy.name) + // Update or create candidate + await findOrUpdate(client, core.space.Model, recruit.class.Vacancy, vacancyId, vacancy) + + const states = await createUpdateSpaceKanban(vacancyId, client, statuses) + + console.log('States generated', vacancy.name) + return { states, vacancyId } +} + +async function createCandidate (_name: string, pos: number, len: number, c: any, client: TxOperations, candId: Ref): Promise { + const names = _name.trim().split(' ') + console.log(`(${pos} pf ${len})`, names[0] + ',', names.slice(1).join(' ')) + + const { sourceFields, telegram, linkedin, github } = parseSocials(c) + + const data: Data = { + name: names.slice(1).join(' ') + ', ' + names[0], + city: get(c, _.city) ?? '', + title: [ + get(c, _.vacancyKind), + get(c, _.area) + ].filter(p => p !== undefined && p.trim().length > 0).filter(onlyUniq).join('/'), + source: [ + get(c, _.socialContacted), + get(c, _.socialChannel), + sourceFields.filter(onlyUniq).join(', ') + ].filter(p => p !== undefined && p.trim().length > 0).filter(onlyUniq).join('/'), + channels: [] + } + + pushChannel(c, data, _.email, contact.channelProvider.Email) + pushChannel(c, data, _.phone, contact.channelProvider.Phone) + + const commentData: string[] = [] + + addComment(commentData, c, _.socialContacted) + addComment(commentData, c, _.recruiter) + addComment(commentData, c, _.requiredPosition) + addComment(commentData, c, _.area2) + addComment(commentData, c, _.area3) + addComment(commentData, c, _.framework) + addComment(commentData, c, _.comment) + + if (telegram !== undefined) { + data.channels.push({ provider: contact.channelProvider.Telegram, value: telegram }) + } + if (linkedin !== undefined) { + data.channels.push({ provider: contact.channelProvider.LinkedIn, value: linkedin }) + } + if (github !== undefined) { + data.channels.push({ provider: contact.channelProvider.GitHub, value: github }) + } + await findOrUpdate(client, recruit.space.CandidatesPublic, recruit.class.Candidate, candId, data) + + const commentId = (candId + '.description.comment') as Ref + + if (commentData.length > 0) { + await findOrUpdateAttached(client, recruit.space.CandidatesPublic, chunter.class.Comment, commentId, { + message: commentData.join('\n
') + }, { attachedTo: candId, attachedClass: recruit.class.Candidate, collection: 'comments' }) + } +} + +function addComment (data: string[], c: any, key: string): void { + const val = get(c, key) + if (val !== undefined) { + data.push(`${key}: ${val}`.replace('\n', '\n
')) + } +} + +function parseSocials (c: any): { sourceFields: string[], telegram: string | undefined, linkedin: string | undefined, github: string | undefined } { + let telegram: string | undefined + let linkedin: string | undefined + let github: string | undefined + + const sourceFields = ([ + get(c, _.socialLink), + get(c, _.socialLink2), + get(c, _.socialLink3), + get(c, _.otherContact), + get(c, _.area2), + get(c, _.area3) + ].filter(p => p !== undefined && p.trim().length > 0) as string[]).filter(t => { + const lc = t.toLocaleLowerCase() + if (lc.startsWith('telegram')) { + telegram = t.substring(8).replace(':', '').trim() + return false + } + if (lc.includes('linkedin.')) { + linkedin = t.trim() + return false + } + if (lc.includes('github.com')) { + github = t.trim() + return false + } + return true + }) + return { sourceFields, telegram, linkedin, github } +} + +function pushChannel (c: any, data: Data, key: string, provider: Ref): void { + const value = get(c, key) + if (value !== undefined) { + data.channels.push({ provider, value }) + } +} +export async function findOrUpdate (client: TxOperations, space: Ref, _class: Ref>, objectId: Ref, data: Data): Promise { + const existingObj = await client.findOne(_class, { _id: objectId, space }) + if (existingObj !== undefined) { + // Check some field changes + const { _id, _class, modifiedOn, modifiedBy, space, ...dta } = existingObj + if (!deepEqual(dta, data)) { + await client.updateDoc(_class, space, objectId, data) + } + } else { + await client.createDoc(_class, space, data, objectId) + } +} + +function randColor (): string { + const letters = '0123456789ABCDEF' + let color = '#' + for (let i = 0; i < 6; i++) { + color += letters[Math.floor(Math.random() * 16)] + } + return color +} + +async function createUpdateSpaceKanban (spaceId: Ref, client: TxOperations, stateNames: string[]): Promise>> { + const states: Map> = new Map() + const stateRanks = genRanks(stateNames.length) + for (const st of stateNames) { + const rank = stateRanks.next().value + + if (rank === undefined) { + console.error('Failed to generate rank') + break + } + + const sid = ('generated-' + spaceId + '.state.' + st.toLowerCase().replace(' ', '_')) as Ref + await findOrUpdate(client, spaceId, task.class.State, + sid, + { + title: st, + color: randColor(), + rank + } + ) + states.set(st, sid) + } + + const doneStates = [ + { class: task.class.WonState, title: 'Won' }, + { class: task.class.LostState, title: 'Lost' } + ] + const doneStateRanks = genRanks(doneStates.length) + for (const st of doneStates) { + const rank = doneStateRanks.next().value + + if (rank === undefined) { + console.error('Failed to generate rank') + break + } + + const sid = ('generated-' + spaceId + '.done-state.' + st.title.toLowerCase().replace(' ', '_')) as Ref + await findOrUpdate(client, spaceId, st.class, + sid, + { + title: st.title, + rank + } + ) + } + + await findOrUpdate(client, spaceId, + task.class.Kanban, + ('generated-' + spaceId + '.kanban') as Ref, + { + attachedTo: spaceId + } + ) + return states +} +async function findOrUpdateAttached (client: TxOperations, space: Ref, _class: Ref>, objectId: Ref, data: AttachedData, attached: {attachedTo: Ref, attachedClass: Ref>, collection: string}): Promise { + let existingObj = await client.findOne(_class, { _id: objectId, space }) as T + if (existingObj !== undefined) { + await client.updateCollection(_class, space, objectId, attached.attachedTo, attached.attachedClass, attached.collection, data as unknown as DocumentUpdate) + } else { + await client.addCollection(_class, space, attached.attachedTo, attached.attachedClass, attached.collection, data, objectId) + existingObj = { _id: objectId, _class, space, ...data, ...attached } as unknown as T + } + return existingObj +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 870992ffd9..134a1fec19 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -31,6 +31,7 @@ import { Client } from 'minio' import { Db, MongoClient } from 'mongodb' import { connect } from './connect' import { rebuildElastic } from './elastic' +import { importXml } from './importer' import { clearTelegramHistory } from './telegram' import { diffWorkspace, dumpWorkspace, initWorkspace, restoreWorkspace, upgradeWorkspace } from './workspace' @@ -247,4 +248,11 @@ program console.log('rebuild end') }) +program + .command('import-xml ') + .description('dump workspace transactions and minio resources') + .action(async (workspace, fileName, cmd) => { + return await importXml(transactorUrl, workspace, minio, fileName, mongodbUri, elasticUrl) + }) + program.parse(process.argv) diff --git a/plugins/view-resources/src/components/Table.svelte b/plugins/view-resources/src/components/Table.svelte index aaebf710a5..d9c54bc296 100644 --- a/plugins/view-resources/src/components/Table.svelte +++ b/plugins/view-resources/src/components/Table.svelte @@ -43,7 +43,7 @@ (result) => { objects = result }, - { sort: { [sortKey]: sortOrder }, ...options } + { sort: { [sortKey]: sortOrder }, ...options, limit: 500 } ) function getValue (doc: Doc, key: string): any { diff --git a/server/elastic/src/adapter.ts b/server/elastic/src/adapter.ts index 1a35c45eaf..d6dbaf4e53 100644 --- a/server/elastic/src/adapter.ts +++ b/server/elastic/src/adapter.ts @@ -26,6 +26,10 @@ class ElasticAdapter implements FullTextAdapter { ) { } + async close (): Promise { + await this.client.close() + } + async search ( search: string ): Promise { @@ -110,7 +114,7 @@ class ElasticAdapter implements FullTextAdapter { /** * @public */ -export async function createElasticAdapter (url: string, dbName: string): Promise { +export async function createElasticAdapter (url: string, dbName: string): Promise Promise}> { const client = new Client({ node: url })