UBERF-7543: Add low level groupBy api and improve security space lookup (#6126)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

packages/core/src/server.ts

@@ -67,6 +67,9 @@ export interface LowLevelStorage {
 
   // Remove a list of documents.
   clean: (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]) => Promise<void>
+
+  // Low level direct group API
+  groupBy: <T>(ctx: MeasureContext, domain: Domain, field: string) => Promise<Set<T>>
 }
 
 export interface Branding {

server/core/src/adapter.ts

@@ -112,6 +112,8 @@ export interface DbAdapter {
   upload: (ctx: MeasureContext, domain: Domain, docs: Doc[]) => Promise<void>
   clean: (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]) => Promise<void>
 
+  groupBy: <T>(ctx: MeasureContext, domain: Domain, field: string) => Promise<Set<T>>
+
   // Bulk update operations
   update: (ctx: MeasureContext, domain: Domain, operations: Map<Ref<Doc>, DocumentUpdate<Doc>>) => Promise<void>
 }

server/core/src/mem.ts

@@ -74,6 +74,10 @@ export class DummyDbAdapter implements DbAdapter {
   async clean (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<void> {}
 
   async update (ctx: MeasureContext, domain: Domain, operations: Map<Ref<Doc>, DocumentUpdate<Doc>>): Promise<void> {}
+
+  async groupBy<T>(ctx: MeasureContext, domain: Domain, field: string): Promise<Set<T>> {
+    return new Set()
+  }
 }
 
 class InMemoryAdapter extends DummyDbAdapter implements DbAdapter {

server/core/src/pipeline.ts

@@ -117,6 +117,12 @@ class PipelineImpl implements Pipeline {
       : await this.storage.findAll(ctx.ctx, _class, query, options)
   }
 
+  async groupBy<T>(ctx: MeasureContext, domain: Domain, field: string): Promise<Set<T>> {
+    return this.head !== undefined
+      ? await this.head.groupBy(ctx, domain, field)
+      : await this.storage.groupBy(ctx, domain, field)
+  }
+
   async searchFulltext (ctx: SessionContext, query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
     return this.head !== undefined
       ? await this.head.searchFulltext(ctx, query, options)

server/core/src/server/storage.ts

@@ -440,6 +440,10 @@ export class TServerStorage implements ServerStorage {
     return this.model.filter((it) => it.modifiedOn > lastModelTx)
   }
 
+  async groupBy<T>(ctx: MeasureContext, domain: Domain, field: string): Promise<Set<T>> {
+    return await this.getAdapter(domain, false).groupBy(ctx, domain, field)
+  }
+
   async findAll<T extends Doc>(
     ctx: MeasureContext,
     clazz: Ref<Class<T>>,

server/core/src/types.ts

@@ -97,6 +97,8 @@ export interface Middleware {
     query: DocumentQuery<T>,
     options?: FindOptions<T>
   ) => Promise<FindResult<T>>
+
+  groupBy: <T>(ctx: MeasureContext, domain: Domain, field: string) => Promise<Set<T>>
   handleBroadcast: HandleBroadcastFunc
   searchFulltext: (ctx: SessionContext, query: SearchQuery, options: SearchOptions) => Promise<SearchResult>
 }

server/elastic/src/backup.ts

@@ -61,6 +61,10 @@ class ElasticDataAdapter implements DbAdapter {
     this.getDocId = (fulltext) => fulltext.slice(0, -1 * (this.workspaceString.length + 1)) as Ref<Doc>
   }
 
+  async groupBy<T>(ctx: MeasureContext, domain: Domain, field: string): Promise<Set<T>> {
+    return new Set()
+  }
+
   async findAll<T extends Doc>(
     ctx: MeasureContext,
     _class: Ref<Class<T>>,

server/middleware/src/base.ts

@@ -23,7 +23,9 @@ import {
   SearchOptions,
   SearchQuery,
   SearchResult,
-  Tx
+  Tx,
+  type Domain,
+  type MeasureContext
 } from '@hcengineering/core'
 import { Middleware, SessionContext, TxMiddlewareResult, type ServerStorage } from '@hcengineering/server-core'
 
@@ -45,6 +47,17 @@ export abstract class BaseMiddleware {
     return await this.provideFindAll(ctx, _class, query, options)
   }
 
+  async providerGroupBy<T>(ctx: MeasureContext, domain: Domain, field: string): Promise<Set<T>> {
+    if (this.next !== undefined) {
+      return await this.next.groupBy(ctx, domain, field)
+    }
+    return await this.storage.groupBy(ctx, domain, field)
+  }
+
+  async groupBy<T>(ctx: MeasureContext, domain: Domain, field: string): Promise<Set<T>> {
+    return await this.providerGroupBy(ctx, domain, field)
+  }
+
   async searchFulltext (ctx: SessionContext, query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
     return await this.provideSearchFulltext(ctx, query, options)
   }

server/middleware/src/spaceSecurity.ts

@@ -450,30 +450,8 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
   }
 
   async loadDomainSpaces (ctx: MeasureContext, domain: Domain): Promise<Set<Ref<Space>>> {
-    const map = new Set<Ref<Space>>()
     const field = this.getKey(domain)
-    while (true) {
-      const nin = Array.from(map.values())
-      const spaces = await this.storage.findAll(
-        ctx,
-        core.class.Doc,
-        nin.length > 0
-          ? {
-              [field]: { $nin: nin }
-            }
-          : {},
-        {
-          projection: { [field]: 1 },
-          limit: 1000,
-          domain
-        }
-      )
-      if (spaces.length === 0) {
-        break
-      }
-      spaces.forEach((p) => map.add((p as any)[field] as Ref<Space>))
-    }
-    return map
+    return await this.storage.groupBy<Ref<Space>>(ctx, domain, field)
   }
 
   async getDomainSpaces (domain: Domain): Promise<Set<Ref<Space>>> {

server/mongo/src/storage.ts

@@ -82,7 +82,8 @@ import {
   type Filter,
   type FindCursor,
   type Sort,
-  type UpdateFilter
+  type UpdateFilter,
+  type FindOptions as MongoFindOptions
 } from 'mongodb'
 import { DBCollectionHelper, getMongoClient, getWorkspaceDB, type MongoClientReference } from './utils'
 
@@ -517,10 +518,18 @@ abstract class MongoAdapterBase implements DbAdapter {
     let result: WithLookup<T>[] = []
     let total = options?.total === true ? 0 : -1
     try {
-      result = await ctx.with('toArray', {}, async (ctx) => await toArray(cursor), {
+      await ctx.with(
+        'toArray',
+        {},
+        async (ctx) => {
+          result = await toArray(cursor)
+        },
+        () => ({
+          size: result.length,
           domain,
           pipeline
         })
+      )
     } catch (e) {
       console.error('error during executing cursor in findWithPipeline', clazz, cutObjectArray(query), options, e)
       throw e
@@ -623,6 +632,33 @@ abstract class MongoAdapterBase implements DbAdapter {
     return false
   }
 
+  @withContext('groupBy')
+  async groupBy<T>(ctx: MeasureContext, domain: Domain, field: string): Promise<Set<T>> {
+    const result = await this.globalCtx.with(
+      'groupBy',
+      { domain },
+      async (ctx) => {
+        const coll = this.collection(domain)
+        const grResult = await coll
+          .aggregate([
+            {
+              $group: {
+                _id: '$' + field
+              }
+            }
+          ])
+          .toArray()
+        return new Set(grResult.map((it) => it._id as unknown as T))
+      },
+
+      () => ({
+        findOps: this.findOps,
+        txOps: this.txOps
+      })
+    )
+    return result
+  }
+
   findOps: number = 0
   txOps: number = 0
   opIndex: number = 0
@@ -695,6 +731,33 @@ abstract class MongoAdapterBase implements DbAdapter {
           const coll = this.collection(domain)
           const mongoQuery = this.translateQuery(_class, query)
 
+          if (options?.limit === 1) {
+            // Skip sort/projection/etc.
+            return await ctx.with(
+              'find-one',
+              {},
+              async (ctx) => {
+                const findOptions: MongoFindOptions = {}
+
+                if (options?.sort !== undefined) {
+                  findOptions.sort = this.collectSort<T>(options, _class)
+                }
+                if (options?.projection !== undefined) {
+                  findOptions.projection = this.calcProjection<T>(options, _class)
+                } else {
+                  findOptions.projection = { '%hash%': 0 }
+                }
+
+                const doc = await coll.findOne(mongoQuery, findOptions)
+                if (doc != null) {
+                  return toFindResult([doc as unknown as T])
+                }
+                return toFindResult([])
+              },
+              { mongoQuery }
+            )
+          }
+
           let cursor = coll.find<T>(mongoQuery)
 
           if (options?.projection !== undefined) {
@@ -702,6 +765,8 @@ abstract class MongoAdapterBase implements DbAdapter {
             if (projection != null) {
               cursor = cursor.project(projection)
             }
+          } else {
+            cursor = cursor.project({ '%hash%': 0 })
           }
           let total: number = -1
           if (options != null) {
@@ -721,11 +786,20 @@ abstract class MongoAdapterBase implements DbAdapter {
 
           // Error in case of timeout
           try {
-            const res: T[] = await ctx.with('toArray', {}, async (ctx) => await toArray(cursor), {
+            let res: T[] = []
+            await ctx.with(
+              'toArray',
+              {},
+              async (ctx) => {
+                res = await toArray(cursor)
+              },
+              () => ({
+                size: res.length,
                 mongoQuery,
                 options,
                 domain
               })
+            )
             if (options?.total === true && options?.limit === undefined) {
               total = res.length
             }

server/server-storage/src/blobStorage.ts

@@ -53,6 +53,10 @@ class StorageBlobAdapter implements DbAdapter {
     return await this.blobAdapter.findAll(ctx, _class, query, options)
   }
 
+  async groupBy<T>(ctx: MeasureContext, domain: Domain, field: string): Promise<Set<T>> {
+    return await this.blobAdapter.groupBy(ctx, domain, field)
+  }
+
   async tx (ctx: MeasureContext, ...tx: Tx[]): Promise<TxResult[]> {
     throw new PlatformError(unknownError('Direct Blob operations are not possible'))
   }

server/ws/src/__tests__/server.test.ts

@@ -72,6 +72,7 @@ describe('server', () => {
       close: async () => {},
       storage: {} as unknown as ServerStorage,
       domains: async () => [],
+      groupBy: async () => new Set(),
       find: (ctx: MeasureContext, domain: Domain) => ({
         next: async (ctx: MeasureContext) => undefined,
         close: async (ctx: MeasureContext) => {}
@@ -170,6 +171,7 @@ describe('server', () => {
           return toFindResult([d as unknown as T])
         },
         tx: async (ctx: SessionContext, tx: Tx): Promise<[TxResult, Tx[], string[] | undefined]> => [{}, [], undefined],
+        groupBy: async () => new Set(),
         close: async () => {},
         storage: {} as unknown as ServerStorage,
         domains: async () => [],