From 4b0e2ceaa70e05148a36e0337b31c72ea7276f15 Mon Sep 17 00:00:00 2001 From: Andrey Platov Date: Tue, 14 Sep 2021 16:21:05 +0200 Subject: [PATCH] update es index Signed-off-by: Andrey Platov --- dev/client-resources/src/connection.ts | 3 ++ dev/server/src/server.ts | 4 ++ server/core/src/fulltext.ts | 48 +++++++++++++------- server/core/src/types.ts | 12 ++++- server/elastic/src/__tests__/adapter.test.ts | 2 +- server/elastic/src/adapter.ts | 29 +++++++++++- 6 files changed, 79 insertions(+), 19 deletions(-) diff --git a/dev/client-resources/src/connection.ts b/dev/client-resources/src/connection.ts index bc77f23339..8b0ff35a9c 100644 --- a/dev/client-resources/src/connection.ts +++ b/dev/client-resources/src/connection.ts @@ -40,6 +40,9 @@ class NullFullTextAdapter implements FullTextAdapter { console.log('noop full text indexer: ', doc) } + async update (id: Ref, update: Record): Promise { + } + async search (query: any): Promise { return [] } diff --git a/dev/server/src/server.ts b/dev/server/src/server.ts index ebab3842c6..fa058f843a 100644 --- a/dev/server/src/server.ts +++ b/dev/server/src/server.ts @@ -15,6 +15,7 @@ // import { DOMAIN_TX } from '@anticrm/core' +import type { Ref, Doc } from '@anticrm/core' import { start as startJsonRpc } from '@anticrm/server-ws' import { createInMemoryAdapter, createInMemoryTxAdapter } from '@anticrm/dev-storage' import { createServerStorage, FullTextAdapter, IndexedDoc } from '@anticrm/server-core' @@ -28,6 +29,9 @@ class NullFullTextAdapter implements FullTextAdapter { console.log('noop full text indexer: ', doc) } + async update (id: Ref, update: Record): Promise { + } + async search (query: any): Promise { return [] } diff --git a/server/core/src/fulltext.ts b/server/core/src/fulltext.ts index 2894760f43..0fb25ed9ad 100644 --- a/server/core/src/fulltext.ts +++ b/server/core/src/fulltext.ts @@ -14,26 +14,14 @@ // limitations under the License. // -import core, { TxCreateDoc, Doc, Ref, Class, Obj, Hierarchy, AnyAttribute, Storage, DocumentQuery, FindOptions, FindResult, TxProcessor } from '@anticrm/core' -import type { AttachedDoc } from '@anticrm/core' +import core, { Hierarchy, AnyAttribute, Storage, DocumentQuery, FindOptions, FindResult, TxProcessor } from '@anticrm/core' +import type { AttachedDoc, TxUpdateDoc, TxCreateDoc, Doc, Ref, Class, Obj } from '@anticrm/core' import type { IndexedDoc, FullTextAdapter, WithFind } from './types' // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const NO_INDEX = [] as AnyAttribute[] -function buildContent (doc: any, attributes: AnyAttribute[]): string { - let result = '' - for (const attr of attributes) { - const value = doc[attr.name] - if (value !== undefined) { - result += value as string - result += ' ' - } - } - return result -} - export class FullTextIndex extends TxProcessor implements Storage { private readonly indexes = new Map>, AnyAttribute[]>() @@ -78,7 +66,7 @@ export class FullTextIndex extends TxProcessor implements Storage { const attributes = this.getFullTextAttributes(tx.objectClass) if (attributes === undefined) return const doc = TxProcessor.createDoc2Doc(tx) - const content = buildContent(doc, attributes) // (doc as any)[attribute.name] + const content = attributes.map(attr => (doc as any)[attr.name]) // buildContent(doc, attributes) // (doc as any)[attribute.name] const indexedDoc: IndexedDoc = { id: doc._id, _class: doc._class, @@ -86,8 +74,36 @@ export class FullTextIndex extends TxProcessor implements Storage { modifiedOn: doc.modifiedOn, space: doc.space, attachedTo: (doc as AttachedDoc).attachedTo, - content + content0: content[0], + content1: content[1], + content2: content[2], + content3: content[3], + content4: content[4], + content5: content[5], + content6: content[6], + content7: content[7], + content8: content[8], + content9: content[9] } return await this.adapter.index(indexedDoc) } + + protected override async txUpdateDoc (tx: TxUpdateDoc): Promise { + const attributes = this.getFullTextAttributes(tx.objectClass) + if (attributes === undefined) return + const ops: any = tx.operations + const update: any = {} + let i = 0 + let shouldUpdate = false + for (const attr of attributes) { + if (ops[attr.name] !== undefined) { + update[`content${i}`] = ops[attr.name] + shouldUpdate = true + i++ + } + } + if (shouldUpdate) { + return await this.adapter.update(tx.objectId, update) + } + } } diff --git a/server/core/src/types.ts b/server/core/src/types.ts index c24613b18b..4e10427340 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -40,7 +40,16 @@ export interface IndexedDoc { modifiedOn: Timestamp modifiedBy: Ref attachedTo?: Ref - content?: string + content0?: string + content1?: string + content2?: string + content3?: string + content4?: string + content5?: string + content6?: string + content7?: string + content8?: string + content9?: string data?: string } @@ -54,6 +63,7 @@ export type SearchQuery = any // TODO: replace with DocumentQuery */ export interface FullTextAdapter { index: (doc: IndexedDoc) => Promise + update: (id: Ref, update: Record) => Promise search: (query: SearchQuery) => Promise } diff --git a/server/elastic/src/__tests__/adapter.test.ts b/server/elastic/src/__tests__/adapter.test.ts index 8ad2c1b700..d72b9d7b57 100644 --- a/server/elastic/src/__tests__/adapter.test.ts +++ b/server/elastic/src/__tests__/adapter.test.ts @@ -27,7 +27,7 @@ describe('client', () => { modifiedBy: 'andrey' as Ref, modifiedOn: 0, space: 'space1' as Ref, - content: 'hey there!' + content0: 'hey there!' } await adapter.index(doc) const hits = await adapter.search({}) diff --git a/server/elastic/src/adapter.ts b/server/elastic/src/adapter.ts index 4329304bab..69df518086 100644 --- a/server/elastic/src/adapter.ts +++ b/server/elastic/src/adapter.ts @@ -14,6 +14,7 @@ // limitations under the License. // +import type { Doc, Ref } from '@anticrm/core' import type { FullTextAdapter, IndexedDoc, SearchQuery } from '@anticrm/server-core' import { Client } from '@elastic/elasticsearch' @@ -34,13 +35,26 @@ class ElasticAdapter implements FullTextAdapter { query: { multi_match: { query: query.$search, - fields: ['content', 'attachment.content'] + fields: [ + 'content0', + 'content1', + 'content2', + 'content3', + 'content4', + 'content5', + 'content6', + 'content7', + 'content8', + 'content9', + 'attachment.content' + ] } } } }) console.log(result) const hits = result.body.hits.hits as any[] + console.log('hits', hits) return hits.map(hit => hit._source) } @@ -49,6 +63,7 @@ class ElasticAdapter implements FullTextAdapter { if (doc.data === undefined) { const resp = await this.client.index({ index: this.db, + id: doc.id, type: '_doc', body: doc }) @@ -58,6 +73,7 @@ class ElasticAdapter implements FullTextAdapter { console.log('attachment pipeline') const resp = await this.client.index({ index: this.db, + id: doc.id, type: '_doc', pipeline: 'attachment', body: doc @@ -66,6 +82,17 @@ class ElasticAdapter implements FullTextAdapter { console.log('error', (resp.meta as any)?.body?.error) } } + + async update (id: Ref, update: Record): Promise { + const resp = await this.client.update({ + index: this.db, + id, + body: { + doc: update + } + }) + console.log('update', resp) + } } /**