Improve mongo lookup pipeline (#1557)

Signed-off-by: Denis Bykhov <80476319+BykhovDenis@users.noreply.github.com>

packages/presentation/package.json

@@ -40,6 +40,7 @@
     "@anticrm/contact": "~0.6.5",
     "@anticrm/login": "~0.6.1",
     "@anticrm/image-cropper": "~0.6.0",
-    "@anticrm/client": "~0.6.2"
+    "@anticrm/client": "~0.6.2",
+    "fast-equals": "^2.0.3"
   }
 }

packages/presentation/src/utils.ts

@@ -22,6 +22,7 @@ import login from '@anticrm/login'
 import { getMetadata } from '@anticrm/platform'
 import { LiveQuery as LQ } from '@anticrm/query'
 import { onDestroy } from 'svelte'
+import { deepEqual } from 'fast-equals'
 
 let liveQuery: LQ
 let client: TxOperations
@@ -57,6 +58,10 @@ export function setClient (_client: Client): void {
 }
 
 export class LiveQuery {
+  private oldClass: Ref<Class<Doc>> | undefined
+  private oldQuery: DocumentQuery<Doc> | undefined
+  private oldOptions: FindOptions<Doc> | undefined
+  private oldCallback: ((result: FindResult<any>) => void) | undefined
   unsubscribe = () => {}
 
   constructor () {
@@ -71,13 +76,34 @@ export class LiveQuery {
     query: DocumentQuery<T>,
     callback: (result: FindResult<T>) => void,
     options?: FindOptions<T>
-  ): void {
+  ): boolean {
+    if (!this.needUpdate(_class, query, callback, options)) {
+      return false
+    }
+    this.oldCallback = callback
+    this.oldClass = _class
+    this.oldOptions = options
+    this.oldQuery = query
     this.unsubscribe()
     const unsub = liveQuery.query(_class, query, callback, options)
     this.unsubscribe = () => {
       unsub()
       this.unsubscribe = () => {}
     }
+    return true
+  }
+
+  private needUpdate<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    callback: (result: FindResult<T>) => void,
+    options?: FindOptions<T>
+  ): boolean {
+    if (!deepEqual(_class, this.oldClass)) return true
+    if (!deepEqual(query, this.oldQuery)) return true
+    if (!deepEqual(callback.toString(), this.oldCallback?.toString())) return true
+    if (!deepEqual(options, this.oldOptions)) return true
+    return false
   }
 }
 

plugins/view-resources/src/components/Table.svelte

@@ -56,7 +56,6 @@
   $: sortingFunction = (config.find((it) => typeof it !== 'string' && it.sortingKey === sortKey) as BuildModelKey)
     ?.sortingFunction
 
-  let qindex = 0
   async function update (
     _class: Ref<Class<Doc>>,
     query: DocumentQuery<Doc>,
@@ -64,16 +63,10 @@
     sortOrder: SortingOrder,
     options?: FindOptions<Doc>
   ) {
-    const c = ++qindex
-    loading = true
-    objects = []
-    q.query(
+    const update = q.query(
       _class,
       query,
       (result) => {
-        if (c !== qindex) {
-          return // our data is invalid.
-        }
         objects = result
         if (sortingFunction !== undefined) {
           const sf = sortingFunction
@@ -82,8 +75,12 @@
         dispatch('content', objects)
         loading = false
       },
-      { sort: { [sortKey]: sortOrder }, ...options, limit: 200 }
+      { sort: { [sortKey]: sortOrder }, limit: 200, ...options }
     )
+    if (update) {
+      objects = []
+      loading = true
+    }
   }
   $: update(_class, query, sortKey, sortOrder, options)
 

server/mongo/src/storage.ts

@@ -46,6 +46,13 @@ function translateDoc (doc: Doc): Document {
   return doc as Document
 }
 
+function isLookupQuery <T extends Doc> (query: DocumentQuery<T>): boolean {
+  for (const key in query) {
+    if (key.includes('$lookup.')) return true
+  }
+  return false
+}
+
 interface LookupStep {
   from: string
   localField: string
@@ -270,49 +277,51 @@ abstract class MongoAdapterBase extends TxProcessor {
     options: FindOptions<T>
   ): Promise<FindResult<T>> {
     const pipeline = []
-    pipeline.push({ $match: this.translateQuery(clazz, query) })
+    const match = { $match: this.translateQuery(clazz, query) }
+    const slowPipeline = isLookupQuery(query)
     const steps = await this.getLookups(options.lookup)
-    for (const step of steps) {
-      pipeline.push({ $lookup: step })
+    if (slowPipeline) {
+      for (const step of steps) {
+        pipeline.push({ $lookup: step })
+      }
     }
+    pipeline.push(match)
+    const resultPipeline: any[] = []
     if (options.sort !== undefined) {
       const sort = {} as any
       for (const _key in options.sort) {
-        let key = _key as string
-        const arr = key.split('.').filter((p) => p)
-        key = ''
-        for (let i = 0; i < arr.length; i++) {
-          const element = arr[i]
-          if (element === '$lookup') {
-            key += arr[++i] + '_lookup'
-          } else {
-            if (!key.endsWith('.') && i > 0) {
-              key += '.'
-            }
-            key += arr[i]
-            if (i !== arr.length - 1) {
-              key += '.'
-            }
-          }
-          // Check if key is belong to mixin class, we need to add prefix.
-          key = this.checkMixinKey<T>(key, clazz)
-        }
+        // Check if key is belong to mixin class, we need to add prefix.
+        const key = this.checkMixinKey<T>(_key, clazz)
         sort[key] = options.sort[_key] === SortingOrder.Ascending ? 1 : -1
       }
-      pipeline.push({ $sort: sort })
+      resultPipeline.push({ $sort: sort })
     }
-    const domain = this.hierarchy.getDomain(clazz)
-    let cursor = this.db.collection(domain).aggregate(pipeline)
-    if (options?.projection !== undefined) {
-      cursor = cursor.project(options.projection)
-    }
-    let result = (await cursor.toArray()) as WithLookup<T>[]
-
-    const total = result.length
     if (options.limit !== undefined) {
-      result = result.slice(0, options.limit)
+      resultPipeline.push({ $limit: options.limit })
     }
-
+    if (!slowPipeline) {
+      for (const step of steps) {
+        resultPipeline.push({ $lookup: step })
+      }
+    }
+    if (options?.projection !== undefined) {
+      resultPipeline.push({ $project: options.projection })
+    }
+    pipeline.push({
+      $facet: {
+        results: resultPipeline,
+        totalCount: [
+          {
+            $count: 'count'
+          }
+        ]
+      }
+    })
+    const domain = this.hierarchy.getDomain(clazz)
+    const cursor = this.db.collection(domain).aggregate(pipeline)
+    const res = (await cursor.toArray())[0]
+    const result = res.results as WithLookup<T>[]
+    const total = res.totalCount?.shift()?.count
     for (const row of result) {
       row.$lookup = {}
       await this.fillLookupValue(options.lookup, row)