From 98a43a98bc4fb071f3e6d9dd25b6587d63ffd3da Mon Sep 17 00:00:00 2001
From: Anna No <anna.no@xored.com>
Date: Thu, 24 Aug 2023 21:00:32 +0700
Subject: [PATCH] EZQMS-106: add elastic search by refs support (#3629)
Signed-off-by: Anna No <anna.no@xored.com>
---
packages/core/src/classes.ts | 8 +++++
server/core/src/indexer/fulltextPush.ts | 45 +++++++++++++++++++++++++
server/core/src/indexer/indexer.ts | 2 +-
server/core/src/indexer/types.ts | 2 +-
4 files changed, 55 insertions(+), 2 deletions(-)
diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts
index 60f430c3d5..5d05086a27 100644
--- a/packages/core/src/classes.ts
+++ b/packages/core/src/classes.ts
@@ -94,7 +94,15 @@ export interface Type<T extends PropertyType> extends UXObject {}
* @public
*/
export enum IndexKind {
+ /**
+ * Attribute with this index annotation should be added to elastic for search
+ * Could be added to string or Ref attribute
+ * TODO: rename properly for better code readability
+ */
FullText,
+ /**
+ * For attribute with this annotation should be created an index in mongo database
+ */
Indexed
}
diff --git a/server/core/src/indexer/fulltextPush.ts b/server/core/src/indexer/fulltextPush.ts
index 7fe34280fe..6dbefa021a 100644
--- a/server/core/src/indexer/fulltextPush.ts
+++ b/server/core/src/indexer/fulltextPush.ts
@@ -14,12 +14,15 @@
//
import core, {
+ AnyAttribute,
+ ArrOf,
Class,
Doc,
DocIndexState,
DocumentQuery,
DocumentUpdate,
extractDocKey,
+ IndexKind,
MeasureContext,
Ref,
ServerStorage,
@@ -95,6 +98,43 @@ export class FullTextPushStage implements FullTextPipelineStage {
return { docs: [], pass: true }
}
+ async indexRefAttributes (
+ attributes: Map<string, AnyAttribute>,
+ doc: DocIndexState,
+ elasticDoc: IndexedDoc,
+ metrics: MeasureContext
+ ): Promise<void> {
+ for (const attribute in doc.attributes) {
+ const { attr } = extractDocKey(attribute)
+ const attrObj = attributes.get(attr)
+ if (
+ attrObj !== null &&
+ attrObj !== undefined &&
+ attrObj.index === IndexKind.FullText &&
+ (attrObj.type._class === core.class.RefTo ||
+ (attrObj.type._class === core.class.ArrOf && (attrObj.type as ArrOf<any>).of._class === core.class.RefTo))
+ ) {
+ const attrStringValue = doc.attributes[attribute]
+ if (attrStringValue !== undefined && attrStringValue !== null && attrStringValue !== '') {
+ const refs = attrStringValue.split(',')
+ const refDocs = await metrics.with(
+ 'ref-docs',
+ {},
+ async (ctx) =>
+ await this.dbStorage.findAll(ctx, core.class.DocIndexState, {
+ _id: { $in: refs }
+ })
+ )
+ if (refDocs.length > 0) {
+ refDocs.forEach((c) => {
+ updateDoc2Elastic(c.attributes, elasticDoc, c._id)
+ })
+ }
+ }
+ }
+ }
+ }
+
async collect (toIndex: DocIndexState[], pipeline: FullTextPipeline, metrics: MeasureContext): Promise<void> {
const bulk: IndexedDoc[] = []
@@ -162,6 +202,11 @@ export class FullTextPushStage implements FullTextPipelineStage {
}
}
+ const allAttributes = pipeline.hierarchy.getAllAttributes(elasticDoc._class)
+
+ // Include child ref attributes
+ await this.indexRefAttributes(allAttributes, doc, elasticDoc, metrics)
+
this.checkIntegrity(elasticDoc)
bulk.push(elasticDoc)
} catch (err: any) {
diff --git a/server/core/src/indexer/indexer.ts b/server/core/src/indexer/indexer.ts
index 1f9ab9d2a5..cd0936a9ea 100644
--- a/server/core/src/indexer/indexer.ts
+++ b/server/core/src/indexer/indexer.ts
@@ -534,7 +534,7 @@ export class FullTextIndexPipeline implements FullTextPipeline {
async checkIndexConsistency (dbStorage: ServerStorage): Promise<void> {
if (process.env.MODEL_VERSION !== undefined) {
- const modelVersion = await (await this.model.findAll(core.class.Version, {})).shift()
+ const modelVersion = (await this.model.findAll(core.class.Version, {})).shift()
if (modelVersion !== undefined) {
const modelVersionString = versionToString(modelVersion)
if (modelVersionString !== process.env.MODEL_VERSION) {
diff --git a/server/core/src/indexer/types.ts b/server/core/src/indexer/types.ts
index 82fca974ba..1763c61594 100644
--- a/server/core/src/indexer/types.ts
+++ b/server/core/src/indexer/types.ts
@@ -107,4 +107,4 @@ export const fieldStateId = 'fld-v5'
/**
* @public
*/
-export const fullTextPushStageId = 'fts-v4'
+export const fullTextPushStageId = 'fts-v5'