UBERF-7428: Fix memory issues (#5940)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

common/config/rush/pnpm-lock.yaml

@@ -962,9 +962,6 @@ dependencies:
   '@tiptap/extension-mention':
     specifier: ^2.2.4
     version: 2.2.4(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)(@tiptap/suggestion@2.2.4)
-  '@tiptap/extension-paragraph':
-    specifier: ^2.2.4
-    version: 2.2.4(@tiptap/core@2.2.4)
   '@tiptap/extension-placeholder':
     specifier: ^2.2.4
     version: 2.2.4(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)
@@ -1122,8 +1119,8 @@ dependencies:
     specifier: 4.21.5
     version: 4.21.5
   bson:
-    specifier: ^6.3.0
+    specifier: ^6.8.0
-    version: 6.3.0
+    version: 6.8.0
   bufferutil:
     specifier: ^4.0.7
     version: 4.0.8
@@ -1302,8 +1299,8 @@ dependencies:
     specifier: ^8.0.0
     version: 8.0.0
   mongodb:
-    specifier: ^6.3.0
+    specifier: ^6.8.0
-    version: 6.3.0
+    version: 6.8.0
   morgan:
     specifier: ^1.10.0
     version: 1.10.0
@@ -4703,6 +4700,12 @@ packages:
       sparse-bitfield: 3.0.3
     dev: false
 
+  /@mongodb-js/saslprep@1.1.7:
+    resolution: {integrity: sha512-dCHW/oEX0KJ4NjDULBo3JiOaK5+6axtpBbS+ao2ZInoAL9/YRQLhXzSNAFz7hP4nzLkIqsfYAK/PDE3+XHny0Q==}
+    dependencies:
+      sparse-bitfield: 3.0.3
+    dev: false
+
   /@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2:
     resolution: {integrity: sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==}
     cpu: [arm64]
@@ -9114,6 +9117,11 @@ packages:
     engines: {node: '>=16.20.1'}
     dev: false
 
+  /bson@6.8.0:
+    resolution: {integrity: sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ==}
+    engines: {node: '>=16.20.1'}
+    dev: false
+
   /buffer-crc32@0.2.13:
     resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==}
     dev: false
@@ -14275,6 +14283,38 @@ packages:
       mongodb-connection-string-url: 3.0.0
     dev: false
 
+  /mongodb@6.8.0:
+    resolution: {integrity: sha512-HGQ9NWDle5WvwMnrvUxsFYPd3JEbqD3RgABHBQRuoCEND0qzhsd0iH5ypHsf1eJ+sXmvmyKpP+FLOKY8Il7jMw==}
+    engines: {node: '>=16.20.1'}
+    peerDependencies:
+      '@aws-sdk/credential-providers': ^3.188.0
+      '@mongodb-js/zstd': ^1.1.0
+      gcp-metadata: ^5.2.0
+      kerberos: ^2.0.1
+      mongodb-client-encryption: '>=6.0.0 <7'
+      snappy: ^7.2.2
+      socks: ^2.7.1
+    peerDependenciesMeta:
+      '@aws-sdk/credential-providers':
+        optional: true
+      '@mongodb-js/zstd':
+        optional: true
+      gcp-metadata:
+        optional: true
+      kerberos:
+        optional: true
+      mongodb-client-encryption:
+        optional: true
+      snappy:
+        optional: true
+      socks:
+        optional: true
+    dependencies:
+      '@mongodb-js/saslprep': 1.1.7
+      bson: 6.8.0
+      mongodb-connection-string-url: 3.0.0
+    dev: false
+
   /monitor-event-loop-delay@1.0.0:
     resolution: {integrity: sha512-YRIr1exCIfBDLZle8WHOfSo7Xg3M+phcZfq9Fx1L6Abo+atGp7cge5pM7PjyBn4s1oZI/BRD4EMrzQBbPpVb5Q==}
     dev: false
@@ -18517,7 +18557,7 @@ packages:
     dev: false
 
   file:projects/account-service.tgz:
-    resolution: {integrity: sha512-odoY6+BB+uYimKUlrE0R6U1U5Bjhlj0XfS9dqgLp1A3lgNipoUN3fWz5/uLEHWiW98dFD5uMGDgwqA6E2zV6CQ==, tarball: file:projects/account-service.tgz}
+    resolution: {integrity: sha512-6mRpNCi4FRpm+ks+wuuSDxGFA9Zvjn++OdmqzbwuOMYj6veUSWc3xwDNV/sf3uoqr9+XvGmcJ97iSIkmtGu+TA==, tarball: file:projects/account-service.tgz}
     name: '@rush-temp/account-service'
     version: 0.0.0
     dependencies:
@@ -18543,7 +18583,7 @@ packages:
       koa-bodyparser: 4.4.1
       koa-morgan: 1.0.1
       koa-router: 12.0.1
-      mongodb: 6.3.0
+      mongodb: 6.8.0
       prettier: 3.2.5
       ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
       ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
@@ -18567,7 +18607,7 @@ packages:
     dev: false
 
   file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3):
-    resolution: {integrity: sha512-yDFeuRyxP9j9kjNWqXpm15LTru2NtfkG5IAgPDjL9twaxPA5GFGVmYBgtRnIKH6HFEZ/GGg/X4OFhLKEXftqhA==, tarball: file:projects/account.tgz}
+    resolution: {integrity: sha512-JTnXiO6DfjRlbofOhtUcTRVJbvTjlVGRk+n1uFvLhQrH0JtaGxlpeeXRAtc49yVlMGr6PuEsiw64yl/HvXuIGQ==, tarball: file:projects/account.tgz}
     id: file:projects/account.tgz
     name: '@rush-temp/account'
     version: 0.0.0
@@ -18583,7 +18623,7 @@ packages:
       eslint-plugin-n: 15.7.0(eslint@8.56.0)
       eslint-plugin-promise: 6.1.1(eslint@8.56.0)
       jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
-      mongodb: 6.3.0
+      mongodb: 6.8.0
       node-fetch: 2.7.0
       prettier: 3.2.5
       ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
@@ -18892,7 +18932,7 @@ packages:
     dev: false
 
   file:projects/auth-providers.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2):
-    resolution: {integrity: sha512-4tWfWzmcPRnHOjt/ehG45iipHiGpDU/DTo++Ryn1L2XP2EiR3QbGoxwnndj+PUT9he+yz5cEwhqUWamI9eUyQA==, tarball: file:projects/auth-providers.tgz}
+    resolution: {integrity: sha512-PDpwpxlOBO4EtMwLUAdsxCl7tOsiiPSA5ob/54WvKYhaoADxg9VAZAEcuyPZgvjnRQFOiRicaSjtNl8gKvfhGw==, tarball: file:projects/auth-providers.tgz}
     id: file:projects/auth-providers.tgz
     name: '@rush-temp/auth-providers'
     version: 0.0.0
@@ -18918,7 +18958,7 @@ packages:
       koa-passport: 6.0.0
       koa-router: 12.0.1
       koa-session: 6.4.0
-      mongodb: 6.3.0
+      mongodb: 6.8.0
       passport-github2: 0.1.12
       passport-google-oauth20: 2.0.0
       prettier: 3.2.5
@@ -19546,7 +19586,7 @@ packages:
     dev: false
 
   file:projects/collaborator.tgz(@tiptap/pm@2.2.4)(bufferutil@4.0.8)(prosemirror-model@1.19.4)(utf-8-validate@6.0.3):
-    resolution: {integrity: sha512-yJ1/DLdH3G4e0CN73FplVGAE5smf+ajGx9AEu6DsfH3AAt7BWyzRqko0Ruhg+p8oqQloOXDnsEW+wWGEvfir1g==, tarball: file:projects/collaborator.tgz}
+    resolution: {integrity: sha512-RHYlR+704EhdJDPooKU+D4BzSDC0Ov00TritI4GOFPnTBbbVkQHCWLqowr1om/rTE/lJ3oHTGPwMBkkfNvAkpw==, tarball: file:projects/collaborator.tgz}
     id: file:projects/collaborator.tgz
     name: '@rush-temp/collaborator'
     version: 0.0.0
@@ -19576,7 +19616,7 @@ packages:
       eslint-plugin-promise: 6.1.1(eslint@8.56.0)
       express: 4.18.3
       jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
-      mongodb: 6.3.0
+      mongodb: 6.8.0
       prettier: 3.2.5
       ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
       ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
@@ -22571,7 +22611,7 @@ packages:
     dev: false
 
   file:projects/mongo.tgz(esbuild@0.20.1)(ts-node@10.9.2):
-    resolution: {integrity: sha512-EDE13Vc6CSTARNf3Iz3F2hyTzJeZxqbh9H/+lnFoP9c2qnuf9xFiYVbvTY+Id8uiA7wRZscXJctVX3j+I4Z7iw==, tarball: file:projects/mongo.tgz}
+    resolution: {integrity: sha512-KZpBO9U1lb9A5pxEVDhez06AEKqg9YKivXmefDmsW53CBgqV9so2eprzgsRlxIQvPz/aMTI7s0bMGL9WUjseRQ==, tarball: file:projects/mongo.tgz}
     id: file:projects/mongo.tgz
     name: '@rush-temp/mongo'
     version: 0.0.0
@@ -22580,14 +22620,14 @@ packages:
       '@types/node': 20.11.19
       '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
       '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
-      bson: 6.3.0
+      bson: 6.8.0
       eslint: 8.56.0
       eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3)
       eslint-plugin-import: 2.29.1(eslint@8.56.0)
       eslint-plugin-n: 15.7.0(eslint@8.56.0)
       eslint-plugin-promise: 6.1.1(eslint@8.56.0)
       jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
-      mongodb: 6.3.0
+      mongodb: 6.8.0
       prettier: 3.2.5
       prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11)
       ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
@@ -22854,7 +22894,7 @@ packages:
     dev: false
 
   file:projects/pod-account.tgz:
-    resolution: {integrity: sha512-UhnxVrB84T40bimL1lr3Hli/dEfC9K3Y0QYBSUFpeNBx65UO59xq+unu77ej1Ckzbs7GebTvpyZ2kLoKVVnCwQ==, tarball: file:projects/pod-account.tgz}
+    resolution: {integrity: sha512-R8dx0uU9jPZ0AmS23UqoxKen0+94WDDHy8YfwhIVuZge+FMUwSBR/7nKBry0bNIyHA2C8yl8AxfJa/o2YedxzQ==, tarball: file:projects/pod-account.tgz}
     name: '@rush-temp/pod-account'
     version: 0.0.0
     dependencies:
@@ -22878,7 +22918,7 @@ packages:
       koa: 2.15.0
       koa-bodyparser: 4.4.1
       koa-router: 12.0.1
-      mongodb: 6.3.0
+      mongodb: 6.8.0
       prettier: 3.2.5
       prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11)
       ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
@@ -23019,7 +23059,7 @@ packages:
     dev: false
 
   file:projects/pod-server.tgz:
-    resolution: {integrity: sha512-Vy2bP0Fg4MFSkPnvyF22PH478ApgBINqguIOp0CJFsYsGYxv90V53Iahb4qh91qvnPXFrnvKf/+HlsHvuQ/0Bg==, tarball: file:projects/pod-server.tgz}
+    resolution: {integrity: sha512-wPRlkGQX23QexM6xt1m5VtJVyxJ4TlT87nVR6Ibf/ty1nrpKTqCYTS/7xxPB6cSC/S8hCb1VheKNkR4+vXXM0Q==, tarball: file:projects/pod-server.tgz}
     name: '@rush-temp/pod-server'
     version: 0.0.0
     dependencies:
@@ -25554,7 +25594,7 @@ packages:
     dev: false
 
   file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3):
-    resolution: {integrity: sha512-jeVFP2yShqCe43VHZ/JqRQNQ7aodz0hc+K5KjeHwGqQsgVwPNjoGiJ+oTn95J8Q4SRjK7B99a5IRUPrPv4QxnA==, tarball: file:projects/server-tool.tgz}
+    resolution: {integrity: sha512-ErBexvPUdBHDpLANxgrqNef832SQXZicJp7U+UDiqr9EsTjKjJ71ZgTYlGtj7unI6XfBEoUlH19cAu7ExV3veA==, tarball: file:projects/server-tool.tgz}
     id: file:projects/server-tool.tgz
     name: '@rush-temp/server-tool'
     version: 0.0.0
@@ -25570,7 +25610,7 @@ packages:
       eslint-plugin-promise: 6.1.1(eslint@8.56.0)
       fast-equals: 5.0.1
       jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
-      mongodb: 6.3.0
+      mongodb: 6.8.0
       prettier: 3.2.5
       ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
       typescript: 5.3.3
@@ -26649,7 +26689,7 @@ packages:
     dev: false
 
   file:projects/text-editor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(prosemirror-model@1.19.4)(ts-node@10.9.2)(utf-8-validate@6.0.3):
-    resolution: {integrity: sha512-3aKZP0apJWozq0ASwe8mC88l4hBYcng8TcFcs1V3iIZe1VIeXWjbsq/6uvLv9XxXZ3AkYyv8O3/aq3k+ryFxRQ==, tarball: file:projects/text-editor.tgz}
+    resolution: {integrity: sha512-5pyb9OAuqNH5EmM6Rd84uVwSbvRgHkix21JoILDHzeAxmRmJk9yfYD7gXGq9nO8iMIawt0Uz4v0pLNvgQAUX6Q==, tarball: file:projects/text-editor.tgz}
     id: file:projects/text-editor.tgz
     name: '@rush-temp/text-editor'
     version: 0.0.0
@@ -26725,6 +26765,7 @@ packages:
       - node-notifier
       - postcss
       - postcss-load-config
+      - prosemirror-inputrules
       - prosemirror-model
       - prosemirror-state
       - prosemirror-view
@@ -26738,7 +26779,7 @@ packages:
     dev: false
 
   file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3):
-    resolution: {integrity: sha512-uaQ+Vil4GXGbC16mXmAz/h3Iwy6QNfCOr85nf/C1c5Zwg0h1rNvC9bu+8sYGXfFGfLeqEo/4l/NrQkZ7/QRVJg==, tarball: file:projects/text.tgz}
+    resolution: {integrity: sha512-mUWWTKZWMvUURAcxKRJaO/G5XdwdP7eAOACU3VU2qA2/RTtHGCREgKMpFFK/LwXXqwx0x+LBwfuozf6jLdyjBg==, tarball: file:projects/text.tgz}
     id: file:projects/text.tgz
     name: '@rush-temp/text'
     version: 0.0.0
@@ -26957,7 +26998,7 @@ packages:
     dev: false
 
   file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.3):
-    resolution: {integrity: sha512-tGU/3qdh3K0tpzxidNbITJoDtEWk6BQu7uxkCjR52j8fwpNq1rx62CYkNz6yzUda+RWKjLjMFivrpBcmLJ2s4w==, tarball: file:projects/tool.tgz}
+    resolution: {integrity: sha512-F0XCwnJslL0DElYI9UUk1wMDaby1XQWa4pP7g8OjdAawPHI7VfhpyIO+IKMUuZiKn58pYFtWW4sNMAJwAEOlBg==, tarball: file:projects/tool.tgz}
     id: file:projects/tool.tgz
     name: '@rush-temp/tool'
     version: 0.0.0
@@ -26986,7 +27027,7 @@ packages:
       jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
       libphonenumber-js: 1.10.56
       mime-types: 2.1.35
-      mongodb: 6.3.0
+      mongodb: 6.8.0
       prettier: 3.2.5
       ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
       ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)

dev/tool/package.json

@@ -144,7 +144,7 @@
     "got": "^11.8.3",
     "libphonenumber-js": "^1.9.46",
     "mime-types": "~2.1.34",
-    "mongodb": "^6.3.0",
+    "mongodb": "^6.8.0",
     "ws": "^8.16.0"
   }
 }

dev/tool/src/benchmark.ts

@@ -38,6 +38,7 @@ import { Worker, isMainThread, parentPort } from 'worker_threads'
 import { CSVWriter } from './csv'
 
 interface StartMessage {
+  email: string
   workspaceId: WorkspaceId
   transactorUrl: string
   id: number
@@ -54,6 +55,7 @@ interface StartMessage {
     // If enabled, will perform write tx for same values and Derived format.
     write: boolean
     sleep: number
+    mode: 'find-all' | 'connect-only'
   }
   binary: boolean
   compression: boolean
@@ -75,8 +77,17 @@ interface PendingMsg extends Msg {
 
 export async function benchmark (
   workspaceId: WorkspaceId[],
+  users: Map<string, string[]>,
   transactorUrl: string,
-  cmd: { from: number, steps: number, sleep: number, binary: boolean, write: boolean, compression: boolean }
+  cmd: {
+    from: number
+    steps: number
+    sleep: number
+    binary: boolean
+    write: boolean
+    compression: boolean
+    mode: 'find-all' | 'connect-only'
+  }
 ): Promise<void> {
   const operating = new Set<string>()
   const workers: Worker[] = []
@@ -278,7 +289,9 @@ export async function benchmark (
             .map((it) => {
               const wsid = workspaceId[randNum(workspaceId.length)]
               const workId = 'w-' + i + '-' + it
+              const wsUsers = users.get(wsid.name) ?? []
               const msg: StartMessage = {
+                email: wsUsers[randNum(wsUsers.length)],
                 workspaceId: wsid,
                 transactorUrl,
                 id: i,
@@ -293,7 +306,8 @@ export async function benchmark (
                     rand: 512
                   },
                   sleep: cmd.sleep,
-                  write: cmd.write
+                  write: cmd.write,
+                  mode: cmd.mode
                 },
                 binary: cmd.binary,
                 compression: cmd.compression
@@ -339,81 +353,91 @@ export function benchmarkWorker (): void {
       setMetadata(client.metadata.UseProtocolCompression, msg.compression)
       console.log('connecting to', msg.workspaceId)
 
-      connection = await connect(msg.transactorUrl, msg.workspaceId, undefined)
+      connection = await connect(msg.transactorUrl, msg.workspaceId, msg.email)
-      const opt = new TxOperations(connection, (core.account.System + '_benchmark') as Ref<Account>)
-      parentPort?.postMessage({
-        type: 'operate',
-        workId: msg.workId
-      })
 
-      const rateLimiter = new RateLimiter(msg.options.rate)
+      if (msg.options.mode === 'find-all') {
-
+        const opt = new TxOperations(connection, (core.account.System + '_benchmark') as Ref<Account>)
-      let bigRunning = 0
-
-      while (msg.options.smallRequests + msg.options.bigRequests > 0) {
-        const variant = Math.random()
-        // console.log(`Thread ${msg.workId} ${msg.options.smallRequests} ${msg.options.bigRequests}`)
-        if (msg.options.bigRequests > 0 && variant < 0.5 && bigRunning === 0) {
-          await rateLimiter.add(async () => {
-            bigRunning = 1
-            await connection?.findAll(core.class.BenchmarkDoc, {
-              source: msg.workId,
-              request: {
-                documents: {
-                  from: 1024,
-                  to: 1000
-                },
-                size: {
-                  // 1kb to 5kb
-                  from: 1 * 1024,
-                  to: 4 * 1024
-                }
-              }
-            })
-          })
-          bigRunning = 0
-          msg.options.bigRequests--
-        }
-        if (msg.options.smallRequests > 0 && variant > 0.5) {
-          await rateLimiter.add(async () => {
-            await connection?.findAll(core.class.BenchmarkDoc, {
-              source: msg.workId,
-              request: {
-                documents: {
-                  from: msg.options.limit.min,
-                  to: msg.options.limit.min + msg.options.limit.rand
-                },
-                size: {
-                  from: 4,
-                  to: 16
-                }
-              }
-            })
-          })
-          msg.options.smallRequests--
-        }
-        if (msg.options.write) {
-          await opt.updateDoc(core.class.BenchmarkDoc, core.space.DerivedTx, '' as Ref<BenchmarkDoc>, {
-            response: 'qwe'
-          })
-        }
-        if (msg.options.sleep > 0) {
-          await new Promise((resolve) => setTimeout(resolve, randNum(msg.options.sleep)))
-        }
-      }
-
-      // clearInterval(infoInterval)
-      await rateLimiter.waitProcessing()
-      const to1 = setTimeout(() => {
         parentPort?.postMessage({
-          type: 'log',
+          type: 'operate',
-          workId: msg.workId,
+          workId: msg.workId
-          msg: `timeout waiting processing${msg.workId}`
         })
-      }, 5000)
+
-      clearTimeout(to1)
+        const rateLimiter = new RateLimiter(msg.options.rate)
-      //
+
-      // console.log(`${msg.idd} perform complete`)
+        let bigRunning = 0
+
+        while (msg.options.smallRequests + msg.options.bigRequests > 0) {
+          const variant = Math.random()
+          // console.log(`Thread ${msg.workId} ${msg.options.smallRequests} ${msg.options.bigRequests}`)
+          if (msg.options.bigRequests > 0 && variant < 0.5 && bigRunning === 0) {
+            await rateLimiter.add(async () => {
+              bigRunning = 1
+              await connection?.findAll(core.class.BenchmarkDoc, {
+                source: msg.workId,
+                request: {
+                  documents: {
+                    from: 1024,
+                    to: 1000
+                  },
+                  size: {
+                    // 1kb to 5kb
+                    from: 1 * 1024,
+                    to: 4 * 1024
+                  }
+                }
+              })
+            })
+            bigRunning = 0
+            msg.options.bigRequests--
+          }
+          if (msg.options.smallRequests > 0 && variant > 0.5) {
+            await rateLimiter.add(async () => {
+              await connection?.findAll(core.class.BenchmarkDoc, {
+                source: msg.workId,
+                request: {
+                  documents: {
+                    from: msg.options.limit.min,
+                    to: msg.options.limit.min + msg.options.limit.rand
+                  },
+                  size: {
+                    from: 4,
+                    to: 16
+                  }
+                }
+              })
+            })
+            msg.options.smallRequests--
+          }
+          if (msg.options.write) {
+            await opt.updateDoc(core.class.BenchmarkDoc, core.space.DerivedTx, '' as Ref<BenchmarkDoc>, {
+              response: 'qwe'
+            })
+          }
+          if (msg.options.sleep > 0) {
+            await new Promise((resolve) => setTimeout(resolve, randNum(msg.options.sleep)))
+          }
+        }
+
+        // clearInterval(infoInterval)
+        await rateLimiter.waitProcessing()
+        const to1 = setTimeout(() => {
+          parentPort?.postMessage({
+            type: 'log',
+            workId: msg.workId,
+            msg: `timeout waiting processing${msg.workId}`
+          })
+        }, 5000)
+        clearTimeout(to1)
+        //
+        // console.log(`${msg.idd} perform complete`)
+      } else if (msg.options.mode === 'connect-only') {
+        parentPort?.postMessage({
+          type: 'operate',
+          workId: msg.workId
+        })
+        // Just a huge timeout
+        await new Promise<void>((resolve) => setTimeout(resolve, 50000000))
+      }
     } catch (err: any) {
       console.error(msg.workspaceId, err)
     } finally {
@@ -427,6 +451,7 @@ export function benchmarkWorker (): void {
       await connection?.close()
       clearTimeout(to)
     }
+
     parentPort?.postMessage({
       type: 'complete',
       workId: msg.workId

dev/tool/src/index.ts

@@ -1018,6 +1018,7 @@ export function devTool (
     .option('--compression <compression>', 'Use protocol compression', false)
     .option('--write <write>', 'Perform write operations', false)
     .option('--workspaces <workspaces>', 'Workspaces to test on, comma separated', '')
+    .option('--mode <mode>', 'A benchmark mode. Supported values: `find-all`, `connect-only` ', 'find-all')
     .action(
       async (cmd: {
         from: string
@@ -1027,20 +1028,50 @@ export function devTool (
         binary: string
         compression: string
         write: string
+        mode: 'find-all' | 'connect-only'
       }) => {
-        console.log(JSON.stringify(cmd))
+        const { mongodbUri } = prepareTools()
-        await benchmark(
+        await withDatabase(mongodbUri, async (db, client) => {
-          cmd.workspaces.split(',').map((it) => getWorkspaceId(it, productId)),
+          console.log(JSON.stringify(cmd))
-          transactorUrl,
+          if (!['find-all', 'connect-only'].includes(cmd.mode)) {
-          {
+            console.log('wrong mode')
+            return
+          }
+
+          const allWorkspacesPure = Array.from(await listWorkspacesPure(db, productId))
+          const allWorkspaces = new Map(allWorkspacesPure.map((it) => [it.workspace, it]))
+
+          let workspaces = cmd.workspaces
+            .split(',')
+            .map((it) => it.trim())
+            .filter((it) => it.length > 0)
+            .map((it) => getWorkspaceId(it, productId))
+
+          if (cmd.workspaces.length === 0) {
+            workspaces = allWorkspacesPure.map((it) => getWorkspaceId(it.workspace, productId))
+          }
+          const accounts = new Map(Array.from(await listAccounts(db)).map((it) => [it._id.toString(), it.email]))
+
+          const accountWorkspaces = new Map<string, string[]>()
+          for (const ws of workspaces) {
+            const wsInfo = allWorkspaces.get(ws.name)
+            if (wsInfo !== undefined) {
+              accountWorkspaces.set(
+                ws.name,
+                wsInfo.accounts.map((it) => accounts.get(it.toString()) as string)
+              )
+            }
+          }
+          await benchmark(workspaces, accountWorkspaces, transactorUrl, {
             steps: parseInt(cmd.steps),
             from: parseInt(cmd.from),
             sleep: parseInt(cmd.sleep),
             binary: cmd.binary === 'true',
             compression: cmd.compression === 'true',
-            write: cmd.write === 'true'
+            write: cmd.write === 'true',
-          }
+            mode: cmd.mode
-        )
+          })
+        })
       }
     )
   program

packages/core/src/measurements/metrics.ts

@@ -101,8 +101,8 @@ export function childMetrics (root: Metrics, path: string[]): Metrics {
 /**
  * @public
  */
-export function metricsAggregate (m: Metrics): Metrics {
+export function metricsAggregate (m: Metrics, limit: number = -1): Metrics {
-  const ms = aggregateMetrics(m.measurements)
+  const ms = aggregateMetrics(m.measurements, limit)
 
   // Use child overage, if there is no top level value specified.
   const me = Object.entries(ms)
@@ -115,6 +115,21 @@ export function metricsAggregate (m: Metrics): Metrics {
         return p + v.value
       }, 0)
 
+  if (limit !== -1) {
+    // We need to keep only top limit items in ms
+    if (Object.keys(ms).length > 0) {
+      const newMs: typeof ms = {}
+      let added = 0
+      for (const [k, v] of Object.entries(ms)) {
+        newMs[k] = v
+        added++
+        if (added >= limit) {
+          break
+        }
+      }
+    }
+  }
+
   return {
     operations: m.operations,
     measurements: ms,
@@ -124,10 +139,10 @@ export function metricsAggregate (m: Metrics): Metrics {
   }
 }
 
-function aggregateMetrics (m: Record<string, Metrics>): Record<string, Metrics> {
+function aggregateMetrics (m: Record<string, Metrics>, limit: number = -1): Record<string, Metrics> {
   const result: Record<string, Metrics> = {}
   for (const [k, v] of Object.entries(m).sort((a, b) => b[1].value - a[1].value)) {
-    result[k] = metricsAggregate(v)
+    result[k] = metricsAggregate(v, limit)
   }
   return result
 }
@@ -184,7 +199,7 @@ function toString (name: string, m: Metrics, offset: number, length: number): st
  * @public
  */
 export function metricsToString (metrics: Metrics, name = 'System', length: number): string {
-  return toString(name, metricsAggregate(metrics), 0, length)
+  return toString(name, metricsAggregate(metrics, 50), 0, length)
 }
 
 function printMetricsParamsRows (
@@ -234,5 +249,5 @@ function toStringRows (name: string, m: Metrics, offset: number): (number | stri
  * @public
  */
 export function metricsToRows (metrics: Metrics, name = 'System'): (number | string)[][] {
-  return toStringRows(name, metricsAggregate(metrics), 0)
+  return toStringRows(name, metricsAggregate(metrics, 50), 0)
 }

plugins/workbench-resources/src/components/ServerManagerUsers.svelte

@@ -70,7 +70,7 @@
 
 <div class="p-6">
   <div class="flex-row-center">
-    Uniq users: {Object.keys(activeSessions).length}
+    Uniq users: {Object.keys(activeSessions).length} of {data?.statistics?.totalClients} connections
   </div>
   <div class="flex-row-center">
     <CheckBox bind:checked={realUsers} />

pods/account/package.json

@@ -55,7 +55,7 @@
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/auth-providers": "^0.6.0",
     "@hcengineering/core": "^0.6.32",
-    "mongodb": "^6.3.0",
+    "mongodb": "^6.8.0",
     "koa": "^2.13.1",
     "koa-router": "^12.0.1",
     "koa-bodyparser": "^4.3.0",

pods/authProviders/package.json

@@ -45,7 +45,7 @@
     "@types/passport-github2": "^1.2.9"
   },
   "dependencies": {
-    "mongodb": "^6.3.0",
+    "mongodb": "^6.8.0",
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/account": "^0.6.0",
     "passport-google-oauth20": "~2.0.0",

server/account-service/package.json

@@ -47,7 +47,7 @@
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/auth-providers": "^0.6.0",
     "@hcengineering/core": "^0.6.32",
-    "mongodb": "^6.3.0",
+    "mongodb": "^6.8.0",
     "koa": "^2.13.1",
     "koa-router": "^12.0.1",
     "koa-bodyparser": "^4.3.0",

server/account/package.json

@@ -39,7 +39,7 @@
     "@types/jest": "^29.5.5"
   },
   "dependencies": {
-    "mongodb": "^6.3.0",
+    "mongodb": "^6.8.0",
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/contact": "^0.6.24",

server/account/src/operations.ts

@@ -1782,10 +1782,14 @@ export async function removeWorkspace (
   const { workspace, account } = await getWorkspaceAndAccount(ctx, db, productId, email, workspaceId)
 
   // Add account into workspace.
-  await db.collection(WORKSPACE_COLLECTION).updateOne({ _id: workspace._id }, { $pull: { accounts: account._id } })
+  await db
+    .collection<Workspace>(WORKSPACE_COLLECTION)
+    .updateOne({ _id: workspace._id }, { $pull: { accounts: account._id } })
 
   // Add account a workspace
-  await db.collection(ACCOUNT_COLLECTION).updateOne({ _id: account._id }, { $pull: { workspaces: workspace._id } })
+  await db
+    .collection<Account>(ACCOUNT_COLLECTION)
+    .updateOne({ _id: account._id }, { $pull: { workspaces: workspace._id } })
   ctx.info('Workspace removed', { email, workspace })
 }
 

server/collaborator/package.json

@@ -73,7 +73,7 @@
     "@hocuspocus/transformer": "^2.9.0",
     "@tiptap/core": "^2.2.4",
     "@tiptap/html": "^2.2.4",
-    "mongodb": "^6.3.0",
+    "mongodb": "^6.8.0",
     "yjs": "^13.5.52",
     "y-prosemirror": "^1.2.1",
     "express": "^4.18.3",

server/collaborator/src/server.ts

@@ -19,7 +19,6 @@ import { Token, decodeToken } from '@hcengineering/server-token'
 import { ServerKit } from '@hcengineering/text'
 import { Hocuspocus } from '@hocuspocus/server'
 import bp from 'body-parser'
-import compression from 'compression'
 import cors from 'cors'
 import express from 'express'
 import { IncomingMessage, createServer } from 'http'
@@ -58,22 +57,6 @@ export async function start (
   const app = express()
   app.use(cors())
   app.use(bp.json())
-  app.use(
-    compression({
-      filter: (req, res) => {
-        if (req.headers['x-no-compression'] != null) {
-          // don't compress responses with this request header
-          return false
-        }
-
-        // fallback to standard filter function
-        return compression.filter(req, res)
-      },
-      level: 1,
-      memLevel: 9
-    })
-  )
-
   const extensions = [
     ServerKit.configure({
       image: {
@@ -208,22 +191,7 @@ export async function start (
 
   const wss = new WebSocketServer({
     noServer: true,
-    perMessageDeflate: {
+    perMessageDeflate: false
-      zlibDeflateOptions: {
-        chunkSize: 32 * 1024,
-        memLevel: 9,
-        level: 1
-      },
-      zlibInflateOptions: {
-        chunkSize: 32 * 1024,
-        memLevel: 9,
-        level: 1
-      },
-      // Below options specified as default values.
-      concurrencyLimit: 10, // Limits zlib concurrency for perf.
-      threshold: 1024 // Size (in bytes) below which messages
-      // should not be compressed if context takeover is disabled.
-    }
   })
 
   wss.on('connection', (incoming: WebSocket, request: IncomingMessage) => {

server/core/src/adapter.ts

@@ -66,14 +66,14 @@ export interface RawDBAdapter {
     workspace: WorkspaceId,
     domain: Domain,
     query: DocumentQuery<T>,
-    options?: Omit<FindOptions<T>, 'projection' | 'lookup'>
+    options?: Omit<FindOptions<T>, 'projection' | 'lookup' | 'total'>
   ) => Promise<FindResult<T>>
   findStream: <T extends Doc>(
     ctx: MeasureContext,
     workspace: WorkspaceId,
     domain: Domain,
     query: DocumentQuery<T>,
-    options?: Omit<FindOptions<T>, 'projection' | 'lookup'>
+    options?: Omit<FindOptions<T>, 'projection' | 'lookup' | 'total'>
   ) => Promise<RawDBAdapterStream<T>>
   upload: <T extends Doc>(ctx: MeasureContext, workspace: WorkspaceId, domain: Domain, docs: T[]) => Promise<void>
   update: <T extends Doc>(

server/middleware/src/spacePermissions.ts

@@ -109,7 +109,17 @@ export class SpacePermissionsMiddleware extends BaseMiddleware implements Middle
       space,
       spaceType.targetClass
     ) as unknown as RolesAssignment
-    this.assignmentBySpace[space._id] = asMixin
+
+    const allPossibleRoles = this.storage.modelDb.findAllSync(core.class.Role, {})
+    const requiredValues: Record<string, any> = {}
+    for (const role of allPossibleRoles) {
+      const v = asMixin[role._id]
+      if (v !== undefined) {
+        requiredValues[role._id] = asMixin[role._id]
+      }
+    }
+
+    this.assignmentBySpace[space._id] = requiredValues
 
     await this.setPermissions(space._id, await this.getRoles(spaceType._id), asMixin)
   }
@@ -193,7 +203,17 @@ export class SpacePermissionsMiddleware extends BaseMiddleware implements Middle
     // Note: currently the whole assignment is always included into the mixin update
     // so we can just rebuild the permissions
     const assignment: RolesAssignment = mixinDoc.attributes as RolesAssignment
-    this.assignmentBySpace[spaceId] = assignment
+
+    const allPossibleRoles = this.storage.modelDb.findAllSync(core.class.Role, {})
+    const requiredValues: Record<string, any> = {}
+    for (const role of allPossibleRoles) {
+      const v = assignment[role._id]
+      if (v !== undefined) {
+        requiredValues[role._id] = assignment[role._id]
+      }
+    }
+
+    this.assignmentBySpace[spaceId] = requiredValues
 
     this.permissionsBySpace[tx.objectId] = {}
     await this.setPermissions(spaceId, await this.getRoles(spaceType._id), assignment)

server/mongo/package.json

@@ -37,7 +37,7 @@
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/server-core": "^0.6.1",
-    "mongodb": "^6.3.0",
+    "mongodb": "^6.8.0",
-    "bson": "^6.3.0"
+    "bson": "^6.8.0"
   }
 }

server/mongo/src/rawAdapter.ts

@@ -42,7 +42,7 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter {
     workspace: WorkspaceId,
     domain: Domain,
     query: DocumentQuery<T>,
-    options?: Omit<FindOptions<T>, 'projection' | 'lookup'>
+    options?: Omit<FindOptions<T>, 'projection' | 'lookup' | 'total'>
   ): Promise<{
       cursor: FindCursor<T>
       total: number
@@ -54,7 +54,7 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter {
       checkKeys: false
     })
 
-    let total: number = -1
+    const total: number = -1
     if (options != null) {
       if (options.sort !== undefined) {
         const sort = collectSort(options)
@@ -63,9 +63,6 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter {
         }
       }
       if (options.limit !== undefined || typeof query._id === 'string') {
-        if (options.total === true) {
-          total = await coll.countDocuments(query)
-        }
         cursor = cursor.limit(options.limit ?? 1)
       }
     }
@@ -78,9 +75,9 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter {
       workspace: WorkspaceId,
       domain: Domain,
       query: DocumentQuery<T>,
-      options?: Omit<FindOptions<T>, 'projection' | 'lookup'>
+      options?: Omit<FindOptions<T>, 'projection' | 'lookup' | 'total'>
     ): Promise<FindResult<T>> {
-      let { cursor, total } = await ctx.with(
+      const { cursor, total } = await ctx.with(
         'get-cursor',
         {},
         async () => await getCursor(workspace, domain, query, options)
@@ -92,9 +89,6 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter {
           ...query,
           ...options
         })
-        if (options?.total === true && options?.limit === undefined) {
-          total = res.length
-        }
         return toFindResult(res, total)
       } catch (e) {
         console.error('error during executing cursor in findAll', cutObjectArray(query), options, e)
@@ -106,7 +100,7 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter {
       workspace: WorkspaceId,
       domain: Domain,
       query: DocumentQuery<T>,
-      options?: Omit<FindOptions<T>, 'projection' | 'lookup'>
+      options?: Omit<FindOptions<T>, 'projection' | 'lookup' | 'total'>
     ): Promise<RawDBAdapterStream<T>> {
       const { cursor } = await getCursor(workspace, domain, query, options)
 

server/mongo/src/storage.ts

@@ -155,9 +155,11 @@ abstract class MongoAdapterBase implements DbAdapter {
     try {
       const existingIndexes = await this.collection(domain).indexes()
       for (const existingIndex of existingIndexes) {
-        const name: string = existingIndex.name
+        if (existingIndex.name !== undefined) {
-        if (deletePattern.test(name) && !keepPattern.test(name)) {
+          const name: string = existingIndex.name
-          await this.collection(domain).dropIndex(existingIndex.name)
+          if (deletePattern.test(name) && !keepPattern.test(name)) {
+            await this.collection(domain).dropIndex(name)
+          }
         }
       }
     } catch (err: any) {

server/tool/package.json

@@ -38,7 +38,7 @@
     "@types/jest": "^29.5.5"
   },
   "dependencies": {
-    "mongodb": "^6.3.0",
+    "mongodb": "^6.8.0",
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/contact": "^0.6.24",

server/ws/src/server.ts

@@ -23,12 +23,12 @@ import core, {
   versionToString,
   withContext,
   type BaseWorkspaceInfo,
+  type Branding,
+  type BrandingMap,
   type MeasureContext,
   type Tx,
   type TxWorkspaceEvent,
-  type WorkspaceId,
+  type WorkspaceId
-  type Branding,
-  type BrandingMap
 } from '@hcengineering/core'
 import { unknownError, type Status } from '@hcengineering/platform'
 import { type HelloRequest, type HelloResponse, type Request, type Response } from '@hcengineering/rpc'
@@ -260,8 +260,19 @@ class TSessionManager implements SessionManager {
     > {
     const wsString = toWorkspaceString(token.workspace, '@')
 
-    let workspaceInfo =
+    let workspaceInfo: WorkspaceLoginInfo | undefined
-      accountsUrl !== '' ? await this.getWorkspaceInfo(ctx, accountsUrl, rawToken) : this.wsFromToken(token)
+    for (let i = 0; i < 5; i++) {
+      try {
+        workspaceInfo =
+          accountsUrl !== '' ? await this.getWorkspaceInfo(ctx, accountsUrl, rawToken) : this.wsFromToken(token)
+        break
+      } catch (err: any) {
+        if (i === 4) {
+          throw err
+        }
+        await new Promise((resolve) => setTimeout(resolve, 10))
+      }
+    }
 
     if (workspaceInfo?.creating === true && token.email !== systemAccountEmail) {
       // No access to workspace for token.
@@ -302,7 +313,7 @@ class TSessionManager implements SessionManager {
     const workspaceName = workspaceInfo.workspaceName ?? workspaceInfo.workspaceUrl ?? workspaceInfo.workspaceId
     const branding =
       (workspaceInfo.branding !== undefined
-        ? Object.values(this.brandingMap).find((b) => b.key === workspaceInfo.branding)
+        ? Object.values(this.brandingMap).find((b) => b.key === (workspaceInfo as WorkspaceLoginInfo).branding)
         : null) ?? null
 
     if (workspace === undefined) {
@@ -1007,7 +1018,7 @@ export function start (
     opt.pipelineFactory,
     opt.port,
     opt.productId,
-    opt.enableCompression ?? true,
+    opt.enableCompression ?? false,
     opt.accountsUrl,
     opt.externalStorage
   )

server/ws/src/server_http.ts

@@ -250,13 +250,13 @@ export function startHttpServer (
           zlibDeflateOptions: {
             // See zlib defaults.
             chunkSize: 32 * 1024,
-            memLevel: 9,
+            memLevel: 1,
             level: 1
           },
           zlibInflateOptions: {
             chunkSize: 32 * 1024,
             level: 1,
-            memLevel: 9
+            memLevel: 1
           },
           serverNoContextTakeover: true,
           clientNoContextTakeover: true,
@@ -327,7 +327,7 @@ export function startHttpServer (
       } catch (err: any) {
         Analytics.handleError(err)
         if (LOGGING_ENABLED) {
-          ctx.error('message error', err)
+          ctx.error('message error', { err })
         }
       }
     })

server/ws/src/stats.ts

@@ -14,7 +14,7 @@ import { type SessionManager } from './types'
  */
 export function getStatistics (ctx: MeasureContext, sessions: SessionManager, admin: boolean): any {
   const data: Record<string, any> = {
-    metrics: metricsAggregate((ctx as any).metrics),
+    metrics: metricsAggregate((ctx as any).metrics, 50),
     statistics: {
       activeSessions: {}
     }
@@ -40,8 +40,9 @@ export function getStatistics (ctx: MeasureContext, sessions: SessionManager, ad
     }
   }
 
-  data.statistics.memoryUsed = Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100
+  const memU = process.memoryUsage()
-  data.statistics.memoryTotal = Math.round((process.memoryUsage().heapTotal / 1024 / 1024) * 100) / 100
+  data.statistics.memoryUsed = Math.round(((memU.heapUsed + memU.rss) / 1024 / 1024) * 100) / 100
+  data.statistics.memoryTotal = Math.round((memU.heapTotal / 1024 / 1024) * 100) / 100
   data.statistics.cpuUsage = Math.round(os.loadavg()[0] * 100) / 100
   data.statistics.freeMem = Math.round((os.freemem() / 1024 / 1024) * 100) / 100
   data.statistics.totalMem = Math.round((os.totalmem() / 1024 / 1024) * 100) / 100