diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 192bf0d2e0..d506b71450 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -962,9 +962,6 @@ dependencies: '@tiptap/extension-mention': specifier: ^2.2.4 version: 2.2.4(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)(@tiptap/suggestion@2.2.4) - '@tiptap/extension-paragraph': - specifier: ^2.2.4 - version: 2.2.4(@tiptap/core@2.2.4) '@tiptap/extension-placeholder': specifier: ^2.2.4 version: 2.2.4(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4) @@ -1122,8 +1119,8 @@ dependencies: specifier: 4.21.5 version: 4.21.5 bson: - specifier: ^6.3.0 - version: 6.3.0 + specifier: ^6.8.0 + version: 6.8.0 bufferutil: specifier: ^4.0.7 version: 4.0.8 @@ -1302,8 +1299,8 @@ dependencies: specifier: ^8.0.0 version: 8.0.0 mongodb: - specifier: ^6.3.0 - version: 6.3.0 + specifier: ^6.8.0 + version: 6.8.0 morgan: specifier: ^1.10.0 version: 1.10.0 @@ -4703,6 +4700,12 @@ packages: sparse-bitfield: 3.0.3 dev: false + /@mongodb-js/saslprep@1.1.7: + resolution: {integrity: sha512-dCHW/oEX0KJ4NjDULBo3JiOaK5+6axtpBbS+ao2ZInoAL9/YRQLhXzSNAFz7hP4nzLkIqsfYAK/PDE3+XHny0Q==} + dependencies: + sparse-bitfield: 3.0.3 + dev: false + /@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2: resolution: {integrity: sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==} cpu: [arm64] @@ -9114,6 +9117,11 @@ packages: engines: {node: '>=16.20.1'} dev: false + /bson@6.8.0: + resolution: {integrity: sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ==} + engines: {node: '>=16.20.1'} + dev: false + /buffer-crc32@0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} dev: false @@ -14275,6 +14283,38 @@ packages: mongodb-connection-string-url: 3.0.0 dev: false + /mongodb@6.8.0: + resolution: {integrity: sha512-HGQ9NWDle5WvwMnrvUxsFYPd3JEbqD3RgABHBQRuoCEND0qzhsd0iH5ypHsf1eJ+sXmvmyKpP+FLOKY8Il7jMw==} + engines: {node: '>=16.20.1'} + peerDependencies: + '@aws-sdk/credential-providers': ^3.188.0 + '@mongodb-js/zstd': ^1.1.0 + gcp-metadata: ^5.2.0 + kerberos: ^2.0.1 + mongodb-client-encryption: '>=6.0.0 <7' + snappy: ^7.2.2 + socks: ^2.7.1 + peerDependenciesMeta: + '@aws-sdk/credential-providers': + optional: true + '@mongodb-js/zstd': + optional: true + gcp-metadata: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + snappy: + optional: true + socks: + optional: true + dependencies: + '@mongodb-js/saslprep': 1.1.7 + bson: 6.8.0 + mongodb-connection-string-url: 3.0.0 + dev: false + /monitor-event-loop-delay@1.0.0: resolution: {integrity: sha512-YRIr1exCIfBDLZle8WHOfSo7Xg3M+phcZfq9Fx1L6Abo+atGp7cge5pM7PjyBn4s1oZI/BRD4EMrzQBbPpVb5Q==} dev: false @@ -18517,7 +18557,7 @@ packages: dev: false file:projects/account-service.tgz: - resolution: {integrity: sha512-odoY6+BB+uYimKUlrE0R6U1U5Bjhlj0XfS9dqgLp1A3lgNipoUN3fWz5/uLEHWiW98dFD5uMGDgwqA6E2zV6CQ==, tarball: file:projects/account-service.tgz} + resolution: {integrity: sha512-6mRpNCi4FRpm+ks+wuuSDxGFA9Zvjn++OdmqzbwuOMYj6veUSWc3xwDNV/sf3uoqr9+XvGmcJ97iSIkmtGu+TA==, tarball: file:projects/account-service.tgz} name: '@rush-temp/account-service' version: 0.0.0 dependencies: @@ -18543,7 +18583,7 @@ packages: koa-bodyparser: 4.4.1 koa-morgan: 1.0.1 koa-router: 12.0.1 - mongodb: 6.3.0 + mongodb: 6.8.0 prettier: 3.2.5 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) @@ -18567,7 +18607,7 @@ packages: dev: false file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3): - resolution: {integrity: sha512-yDFeuRyxP9j9kjNWqXpm15LTru2NtfkG5IAgPDjL9twaxPA5GFGVmYBgtRnIKH6HFEZ/GGg/X4OFhLKEXftqhA==, tarball: file:projects/account.tgz} + resolution: {integrity: sha512-JTnXiO6DfjRlbofOhtUcTRVJbvTjlVGRk+n1uFvLhQrH0JtaGxlpeeXRAtc49yVlMGr6PuEsiw64yl/HvXuIGQ==, tarball: file:projects/account.tgz} id: file:projects/account.tgz name: '@rush-temp/account' version: 0.0.0 @@ -18583,7 +18623,7 @@ packages: eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) - mongodb: 6.3.0 + mongodb: 6.8.0 node-fetch: 2.7.0 prettier: 3.2.5 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) @@ -18892,7 +18932,7 @@ packages: dev: false file:projects/auth-providers.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-4tWfWzmcPRnHOjt/ehG45iipHiGpDU/DTo++Ryn1L2XP2EiR3QbGoxwnndj+PUT9he+yz5cEwhqUWamI9eUyQA==, tarball: file:projects/auth-providers.tgz} + resolution: {integrity: sha512-PDpwpxlOBO4EtMwLUAdsxCl7tOsiiPSA5ob/54WvKYhaoADxg9VAZAEcuyPZgvjnRQFOiRicaSjtNl8gKvfhGw==, tarball: file:projects/auth-providers.tgz} id: file:projects/auth-providers.tgz name: '@rush-temp/auth-providers' version: 0.0.0 @@ -18918,7 +18958,7 @@ packages: koa-passport: 6.0.0 koa-router: 12.0.1 koa-session: 6.4.0 - mongodb: 6.3.0 + mongodb: 6.8.0 passport-github2: 0.1.12 passport-google-oauth20: 2.0.0 prettier: 3.2.5 @@ -19546,7 +19586,7 @@ packages: dev: false file:projects/collaborator.tgz(@tiptap/pm@2.2.4)(bufferutil@4.0.8)(prosemirror-model@1.19.4)(utf-8-validate@6.0.3): - resolution: {integrity: sha512-yJ1/DLdH3G4e0CN73FplVGAE5smf+ajGx9AEu6DsfH3AAt7BWyzRqko0Ruhg+p8oqQloOXDnsEW+wWGEvfir1g==, tarball: file:projects/collaborator.tgz} + resolution: {integrity: sha512-RHYlR+704EhdJDPooKU+D4BzSDC0Ov00TritI4GOFPnTBbbVkQHCWLqowr1om/rTE/lJ3oHTGPwMBkkfNvAkpw==, tarball: file:projects/collaborator.tgz} id: file:projects/collaborator.tgz name: '@rush-temp/collaborator' version: 0.0.0 @@ -19576,7 +19616,7 @@ packages: eslint-plugin-promise: 6.1.1(eslint@8.56.0) express: 4.18.3 jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) - mongodb: 6.3.0 + mongodb: 6.8.0 prettier: 3.2.5 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) @@ -22571,7 +22611,7 @@ packages: dev: false file:projects/mongo.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-EDE13Vc6CSTARNf3Iz3F2hyTzJeZxqbh9H/+lnFoP9c2qnuf9xFiYVbvTY+Id8uiA7wRZscXJctVX3j+I4Z7iw==, tarball: file:projects/mongo.tgz} + resolution: {integrity: sha512-KZpBO9U1lb9A5pxEVDhez06AEKqg9YKivXmefDmsW53CBgqV9so2eprzgsRlxIQvPz/aMTI7s0bMGL9WUjseRQ==, tarball: file:projects/mongo.tgz} id: file:projects/mongo.tgz name: '@rush-temp/mongo' version: 0.0.0 @@ -22580,14 +22620,14 @@ packages: '@types/node': 20.11.19 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) - bson: 6.3.0 + bson: 6.8.0 eslint: 8.56.0 eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3) eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) - mongodb: 6.3.0 + mongodb: 6.8.0 prettier: 3.2.5 prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11) ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) @@ -22854,7 +22894,7 @@ packages: dev: false file:projects/pod-account.tgz: - resolution: {integrity: sha512-UhnxVrB84T40bimL1lr3Hli/dEfC9K3Y0QYBSUFpeNBx65UO59xq+unu77ej1Ckzbs7GebTvpyZ2kLoKVVnCwQ==, tarball: file:projects/pod-account.tgz} + resolution: {integrity: sha512-R8dx0uU9jPZ0AmS23UqoxKen0+94WDDHy8YfwhIVuZge+FMUwSBR/7nKBry0bNIyHA2C8yl8AxfJa/o2YedxzQ==, tarball: file:projects/pod-account.tgz} name: '@rush-temp/pod-account' version: 0.0.0 dependencies: @@ -22878,7 +22918,7 @@ packages: koa: 2.15.0 koa-bodyparser: 4.4.1 koa-router: 12.0.1 - mongodb: 6.3.0 + mongodb: 6.8.0 prettier: 3.2.5 prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11) ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) @@ -23019,7 +23059,7 @@ packages: dev: false file:projects/pod-server.tgz: - resolution: {integrity: sha512-Vy2bP0Fg4MFSkPnvyF22PH478ApgBINqguIOp0CJFsYsGYxv90V53Iahb4qh91qvnPXFrnvKf/+HlsHvuQ/0Bg==, tarball: file:projects/pod-server.tgz} + resolution: {integrity: sha512-wPRlkGQX23QexM6xt1m5VtJVyxJ4TlT87nVR6Ibf/ty1nrpKTqCYTS/7xxPB6cSC/S8hCb1VheKNkR4+vXXM0Q==, tarball: file:projects/pod-server.tgz} name: '@rush-temp/pod-server' version: 0.0.0 dependencies: @@ -25554,7 +25594,7 @@ packages: dev: false file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3): - resolution: {integrity: sha512-jeVFP2yShqCe43VHZ/JqRQNQ7aodz0hc+K5KjeHwGqQsgVwPNjoGiJ+oTn95J8Q4SRjK7B99a5IRUPrPv4QxnA==, tarball: file:projects/server-tool.tgz} + resolution: {integrity: sha512-ErBexvPUdBHDpLANxgrqNef832SQXZicJp7U+UDiqr9EsTjKjJ71ZgTYlGtj7unI6XfBEoUlH19cAu7ExV3veA==, tarball: file:projects/server-tool.tgz} id: file:projects/server-tool.tgz name: '@rush-temp/server-tool' version: 0.0.0 @@ -25570,7 +25610,7 @@ packages: eslint-plugin-promise: 6.1.1(eslint@8.56.0) fast-equals: 5.0.1 jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) - mongodb: 6.3.0 + mongodb: 6.8.0 prettier: 3.2.5 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) typescript: 5.3.3 @@ -26649,7 +26689,7 @@ packages: dev: false file:projects/text-editor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(prosemirror-model@1.19.4)(ts-node@10.9.2)(utf-8-validate@6.0.3): - resolution: {integrity: sha512-3aKZP0apJWozq0ASwe8mC88l4hBYcng8TcFcs1V3iIZe1VIeXWjbsq/6uvLv9XxXZ3AkYyv8O3/aq3k+ryFxRQ==, tarball: file:projects/text-editor.tgz} + resolution: {integrity: sha512-5pyb9OAuqNH5EmM6Rd84uVwSbvRgHkix21JoILDHzeAxmRmJk9yfYD7gXGq9nO8iMIawt0Uz4v0pLNvgQAUX6Q==, tarball: file:projects/text-editor.tgz} id: file:projects/text-editor.tgz name: '@rush-temp/text-editor' version: 0.0.0 @@ -26725,6 +26765,7 @@ packages: - node-notifier - postcss - postcss-load-config + - prosemirror-inputrules - prosemirror-model - prosemirror-state - prosemirror-view @@ -26738,7 +26779,7 @@ packages: dev: false file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3): - resolution: {integrity: sha512-uaQ+Vil4GXGbC16mXmAz/h3Iwy6QNfCOr85nf/C1c5Zwg0h1rNvC9bu+8sYGXfFGfLeqEo/4l/NrQkZ7/QRVJg==, tarball: file:projects/text.tgz} + resolution: {integrity: sha512-mUWWTKZWMvUURAcxKRJaO/G5XdwdP7eAOACU3VU2qA2/RTtHGCREgKMpFFK/LwXXqwx0x+LBwfuozf6jLdyjBg==, tarball: file:projects/text.tgz} id: file:projects/text.tgz name: '@rush-temp/text' version: 0.0.0 @@ -26957,7 +26998,7 @@ packages: dev: false file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.3): - resolution: {integrity: sha512-tGU/3qdh3K0tpzxidNbITJoDtEWk6BQu7uxkCjR52j8fwpNq1rx62CYkNz6yzUda+RWKjLjMFivrpBcmLJ2s4w==, tarball: file:projects/tool.tgz} + resolution: {integrity: sha512-F0XCwnJslL0DElYI9UUk1wMDaby1XQWa4pP7g8OjdAawPHI7VfhpyIO+IKMUuZiKn58pYFtWW4sNMAJwAEOlBg==, tarball: file:projects/tool.tgz} id: file:projects/tool.tgz name: '@rush-temp/tool' version: 0.0.0 @@ -26986,7 +27027,7 @@ packages: jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) libphonenumber-js: 1.10.56 mime-types: 2.1.35 - mongodb: 6.3.0 + mongodb: 6.8.0 prettier: 3.2.5 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) diff --git a/dev/tool/package.json b/dev/tool/package.json index 1385869c89..be0414b5b6 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -144,7 +144,7 @@ "got": "^11.8.3", "libphonenumber-js": "^1.9.46", "mime-types": "~2.1.34", - "mongodb": "^6.3.0", + "mongodb": "^6.8.0", "ws": "^8.16.0" } } diff --git a/dev/tool/src/benchmark.ts b/dev/tool/src/benchmark.ts index 0a125630af..8d6da2ed94 100644 --- a/dev/tool/src/benchmark.ts +++ b/dev/tool/src/benchmark.ts @@ -38,6 +38,7 @@ import { Worker, isMainThread, parentPort } from 'worker_threads' import { CSVWriter } from './csv' interface StartMessage { + email: string workspaceId: WorkspaceId transactorUrl: string id: number @@ -54,6 +55,7 @@ interface StartMessage { // If enabled, will perform write tx for same values and Derived format. write: boolean sleep: number + mode: 'find-all' | 'connect-only' } binary: boolean compression: boolean @@ -75,8 +77,17 @@ interface PendingMsg extends Msg { export async function benchmark ( workspaceId: WorkspaceId[], + users: Map, transactorUrl: string, - cmd: { from: number, steps: number, sleep: number, binary: boolean, write: boolean, compression: boolean } + cmd: { + from: number + steps: number + sleep: number + binary: boolean + write: boolean + compression: boolean + mode: 'find-all' | 'connect-only' + } ): Promise { const operating = new Set() const workers: Worker[] = [] @@ -278,7 +289,9 @@ export async function benchmark ( .map((it) => { const wsid = workspaceId[randNum(workspaceId.length)] const workId = 'w-' + i + '-' + it + const wsUsers = users.get(wsid.name) ?? [] const msg: StartMessage = { + email: wsUsers[randNum(wsUsers.length)], workspaceId: wsid, transactorUrl, id: i, @@ -293,7 +306,8 @@ export async function benchmark ( rand: 512 }, sleep: cmd.sleep, - write: cmd.write + write: cmd.write, + mode: cmd.mode }, binary: cmd.binary, compression: cmd.compression @@ -339,81 +353,91 @@ export function benchmarkWorker (): void { setMetadata(client.metadata.UseProtocolCompression, msg.compression) console.log('connecting to', msg.workspaceId) - connection = await connect(msg.transactorUrl, msg.workspaceId, undefined) - const opt = new TxOperations(connection, (core.account.System + '_benchmark') as Ref) - parentPort?.postMessage({ - type: 'operate', - workId: msg.workId - }) + connection = await connect(msg.transactorUrl, msg.workspaceId, msg.email) - const rateLimiter = new RateLimiter(msg.options.rate) - - let bigRunning = 0 - - while (msg.options.smallRequests + msg.options.bigRequests > 0) { - const variant = Math.random() - // console.log(`Thread ${msg.workId} ${msg.options.smallRequests} ${msg.options.bigRequests}`) - if (msg.options.bigRequests > 0 && variant < 0.5 && bigRunning === 0) { - await rateLimiter.add(async () => { - bigRunning = 1 - await connection?.findAll(core.class.BenchmarkDoc, { - source: msg.workId, - request: { - documents: { - from: 1024, - to: 1000 - }, - size: { - // 1kb to 5kb - from: 1 * 1024, - to: 4 * 1024 - } - } - }) - }) - bigRunning = 0 - msg.options.bigRequests-- - } - if (msg.options.smallRequests > 0 && variant > 0.5) { - await rateLimiter.add(async () => { - await connection?.findAll(core.class.BenchmarkDoc, { - source: msg.workId, - request: { - documents: { - from: msg.options.limit.min, - to: msg.options.limit.min + msg.options.limit.rand - }, - size: { - from: 4, - to: 16 - } - } - }) - }) - msg.options.smallRequests-- - } - if (msg.options.write) { - await opt.updateDoc(core.class.BenchmarkDoc, core.space.DerivedTx, '' as Ref, { - response: 'qwe' - }) - } - if (msg.options.sleep > 0) { - await new Promise((resolve) => setTimeout(resolve, randNum(msg.options.sleep))) - } - } - - // clearInterval(infoInterval) - await rateLimiter.waitProcessing() - const to1 = setTimeout(() => { + if (msg.options.mode === 'find-all') { + const opt = new TxOperations(connection, (core.account.System + '_benchmark') as Ref) parentPort?.postMessage({ - type: 'log', - workId: msg.workId, - msg: `timeout waiting processing${msg.workId}` + type: 'operate', + workId: msg.workId }) - }, 5000) - clearTimeout(to1) - // - // console.log(`${msg.idd} perform complete`) + + const rateLimiter = new RateLimiter(msg.options.rate) + + let bigRunning = 0 + + while (msg.options.smallRequests + msg.options.bigRequests > 0) { + const variant = Math.random() + // console.log(`Thread ${msg.workId} ${msg.options.smallRequests} ${msg.options.bigRequests}`) + if (msg.options.bigRequests > 0 && variant < 0.5 && bigRunning === 0) { + await rateLimiter.add(async () => { + bigRunning = 1 + await connection?.findAll(core.class.BenchmarkDoc, { + source: msg.workId, + request: { + documents: { + from: 1024, + to: 1000 + }, + size: { + // 1kb to 5kb + from: 1 * 1024, + to: 4 * 1024 + } + } + }) + }) + bigRunning = 0 + msg.options.bigRequests-- + } + if (msg.options.smallRequests > 0 && variant > 0.5) { + await rateLimiter.add(async () => { + await connection?.findAll(core.class.BenchmarkDoc, { + source: msg.workId, + request: { + documents: { + from: msg.options.limit.min, + to: msg.options.limit.min + msg.options.limit.rand + }, + size: { + from: 4, + to: 16 + } + } + }) + }) + msg.options.smallRequests-- + } + if (msg.options.write) { + await opt.updateDoc(core.class.BenchmarkDoc, core.space.DerivedTx, '' as Ref, { + response: 'qwe' + }) + } + if (msg.options.sleep > 0) { + await new Promise((resolve) => setTimeout(resolve, randNum(msg.options.sleep))) + } + } + + // clearInterval(infoInterval) + await rateLimiter.waitProcessing() + const to1 = setTimeout(() => { + parentPort?.postMessage({ + type: 'log', + workId: msg.workId, + msg: `timeout waiting processing${msg.workId}` + }) + }, 5000) + clearTimeout(to1) + // + // console.log(`${msg.idd} perform complete`) + } else if (msg.options.mode === 'connect-only') { + parentPort?.postMessage({ + type: 'operate', + workId: msg.workId + }) + // Just a huge timeout + await new Promise((resolve) => setTimeout(resolve, 50000000)) + } } catch (err: any) { console.error(msg.workspaceId, err) } finally { @@ -427,6 +451,7 @@ export function benchmarkWorker (): void { await connection?.close() clearTimeout(to) } + parentPort?.postMessage({ type: 'complete', workId: msg.workId diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 096ac8630a..eb316e8ff6 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -1018,6 +1018,7 @@ export function devTool ( .option('--compression ', 'Use protocol compression', false) .option('--write ', 'Perform write operations', false) .option('--workspaces ', 'Workspaces to test on, comma separated', '') + .option('--mode ', 'A benchmark mode. Supported values: `find-all`, `connect-only` ', 'find-all') .action( async (cmd: { from: string @@ -1027,20 +1028,50 @@ export function devTool ( binary: string compression: string write: string + mode: 'find-all' | 'connect-only' }) => { - console.log(JSON.stringify(cmd)) - await benchmark( - cmd.workspaces.split(',').map((it) => getWorkspaceId(it, productId)), - transactorUrl, - { + const { mongodbUri } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + console.log(JSON.stringify(cmd)) + if (!['find-all', 'connect-only'].includes(cmd.mode)) { + console.log('wrong mode') + return + } + + const allWorkspacesPure = Array.from(await listWorkspacesPure(db, productId)) + const allWorkspaces = new Map(allWorkspacesPure.map((it) => [it.workspace, it])) + + let workspaces = cmd.workspaces + .split(',') + .map((it) => it.trim()) + .filter((it) => it.length > 0) + .map((it) => getWorkspaceId(it, productId)) + + if (cmd.workspaces.length === 0) { + workspaces = allWorkspacesPure.map((it) => getWorkspaceId(it.workspace, productId)) + } + const accounts = new Map(Array.from(await listAccounts(db)).map((it) => [it._id.toString(), it.email])) + + const accountWorkspaces = new Map() + for (const ws of workspaces) { + const wsInfo = allWorkspaces.get(ws.name) + if (wsInfo !== undefined) { + accountWorkspaces.set( + ws.name, + wsInfo.accounts.map((it) => accounts.get(it.toString()) as string) + ) + } + } + await benchmark(workspaces, accountWorkspaces, transactorUrl, { steps: parseInt(cmd.steps), from: parseInt(cmd.from), sleep: parseInt(cmd.sleep), binary: cmd.binary === 'true', compression: cmd.compression === 'true', - write: cmd.write === 'true' - } - ) + write: cmd.write === 'true', + mode: cmd.mode + }) + }) } ) program diff --git a/packages/core/src/measurements/metrics.ts b/packages/core/src/measurements/metrics.ts index 1d8f8c7882..f87dd470e8 100644 --- a/packages/core/src/measurements/metrics.ts +++ b/packages/core/src/measurements/metrics.ts @@ -101,8 +101,8 @@ export function childMetrics (root: Metrics, path: string[]): Metrics { /** * @public */ -export function metricsAggregate (m: Metrics): Metrics { - const ms = aggregateMetrics(m.measurements) +export function metricsAggregate (m: Metrics, limit: number = -1): Metrics { + const ms = aggregateMetrics(m.measurements, limit) // Use child overage, if there is no top level value specified. const me = Object.entries(ms) @@ -115,6 +115,21 @@ export function metricsAggregate (m: Metrics): Metrics { return p + v.value }, 0) + if (limit !== -1) { + // We need to keep only top limit items in ms + if (Object.keys(ms).length > 0) { + const newMs: typeof ms = {} + let added = 0 + for (const [k, v] of Object.entries(ms)) { + newMs[k] = v + added++ + if (added >= limit) { + break + } + } + } + } + return { operations: m.operations, measurements: ms, @@ -124,10 +139,10 @@ export function metricsAggregate (m: Metrics): Metrics { } } -function aggregateMetrics (m: Record): Record { +function aggregateMetrics (m: Record, limit: number = -1): Record { const result: Record = {} for (const [k, v] of Object.entries(m).sort((a, b) => b[1].value - a[1].value)) { - result[k] = metricsAggregate(v) + result[k] = metricsAggregate(v, limit) } return result } @@ -184,7 +199,7 @@ function toString (name: string, m: Metrics, offset: number, length: number): st * @public */ export function metricsToString (metrics: Metrics, name = 'System', length: number): string { - return toString(name, metricsAggregate(metrics), 0, length) + return toString(name, metricsAggregate(metrics, 50), 0, length) } function printMetricsParamsRows ( @@ -234,5 +249,5 @@ function toStringRows (name: string, m: Metrics, offset: number): (number | stri * @public */ export function metricsToRows (metrics: Metrics, name = 'System'): (number | string)[][] { - return toStringRows(name, metricsAggregate(metrics), 0) + return toStringRows(name, metricsAggregate(metrics, 50), 0) } diff --git a/plugins/workbench-resources/src/components/ServerManagerUsers.svelte b/plugins/workbench-resources/src/components/ServerManagerUsers.svelte index dadb8948b2..47aad15c92 100644 --- a/plugins/workbench-resources/src/components/ServerManagerUsers.svelte +++ b/plugins/workbench-resources/src/components/ServerManagerUsers.svelte @@ -70,7 +70,7 @@
- Uniq users: {Object.keys(activeSessions).length} + Uniq users: {Object.keys(activeSessions).length} of {data?.statistics?.totalClients} connections
diff --git a/pods/account/package.json b/pods/account/package.json index 01d82c77bb..79be7dc399 100644 --- a/pods/account/package.json +++ b/pods/account/package.json @@ -55,7 +55,7 @@ "@hcengineering/platform": "^0.6.11", "@hcengineering/auth-providers": "^0.6.0", "@hcengineering/core": "^0.6.32", - "mongodb": "^6.3.0", + "mongodb": "^6.8.0", "koa": "^2.13.1", "koa-router": "^12.0.1", "koa-bodyparser": "^4.3.0", diff --git a/pods/authProviders/package.json b/pods/authProviders/package.json index b95f83cc8c..1a69c5936a 100644 --- a/pods/authProviders/package.json +++ b/pods/authProviders/package.json @@ -45,7 +45,7 @@ "@types/passport-github2": "^1.2.9" }, "dependencies": { - "mongodb": "^6.3.0", + "mongodb": "^6.8.0", "@hcengineering/core": "^0.6.32", "@hcengineering/account": "^0.6.0", "passport-google-oauth20": "~2.0.0", diff --git a/server/account-service/package.json b/server/account-service/package.json index 358004721c..b95503530e 100644 --- a/server/account-service/package.json +++ b/server/account-service/package.json @@ -47,7 +47,7 @@ "@hcengineering/platform": "^0.6.11", "@hcengineering/auth-providers": "^0.6.0", "@hcengineering/core": "^0.6.32", - "mongodb": "^6.3.0", + "mongodb": "^6.8.0", "koa": "^2.13.1", "koa-router": "^12.0.1", "koa-bodyparser": "^4.3.0", diff --git a/server/account/package.json b/server/account/package.json index 2abe63d6f2..2e1ee60320 100644 --- a/server/account/package.json +++ b/server/account/package.json @@ -39,7 +39,7 @@ "@types/jest": "^29.5.5" }, "dependencies": { - "mongodb": "^6.3.0", + "mongodb": "^6.8.0", "@hcengineering/platform": "^0.6.11", "@hcengineering/core": "^0.6.32", "@hcengineering/contact": "^0.6.24", diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index c57938bf42..9f0358150f 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -1782,10 +1782,14 @@ export async function removeWorkspace ( const { workspace, account } = await getWorkspaceAndAccount(ctx, db, productId, email, workspaceId) // Add account into workspace. - await db.collection(WORKSPACE_COLLECTION).updateOne({ _id: workspace._id }, { $pull: { accounts: account._id } }) + await db + .collection(WORKSPACE_COLLECTION) + .updateOne({ _id: workspace._id }, { $pull: { accounts: account._id } }) // Add account a workspace - await db.collection(ACCOUNT_COLLECTION).updateOne({ _id: account._id }, { $pull: { workspaces: workspace._id } }) + await db + .collection(ACCOUNT_COLLECTION) + .updateOne({ _id: account._id }, { $pull: { workspaces: workspace._id } }) ctx.info('Workspace removed', { email, workspace }) } diff --git a/server/collaborator/package.json b/server/collaborator/package.json index 08a8a7de19..4b80c4b0b8 100644 --- a/server/collaborator/package.json +++ b/server/collaborator/package.json @@ -73,7 +73,7 @@ "@hocuspocus/transformer": "^2.9.0", "@tiptap/core": "^2.2.4", "@tiptap/html": "^2.2.4", - "mongodb": "^6.3.0", + "mongodb": "^6.8.0", "yjs": "^13.5.52", "y-prosemirror": "^1.2.1", "express": "^4.18.3", diff --git a/server/collaborator/src/server.ts b/server/collaborator/src/server.ts index 2cfdcb428e..cf5a456105 100644 --- a/server/collaborator/src/server.ts +++ b/server/collaborator/src/server.ts @@ -19,7 +19,6 @@ import { Token, decodeToken } from '@hcengineering/server-token' import { ServerKit } from '@hcengineering/text' import { Hocuspocus } from '@hocuspocus/server' import bp from 'body-parser' -import compression from 'compression' import cors from 'cors' import express from 'express' import { IncomingMessage, createServer } from 'http' @@ -58,22 +57,6 @@ export async function start ( const app = express() app.use(cors()) app.use(bp.json()) - app.use( - compression({ - filter: (req, res) => { - if (req.headers['x-no-compression'] != null) { - // don't compress responses with this request header - return false - } - - // fallback to standard filter function - return compression.filter(req, res) - }, - level: 1, - memLevel: 9 - }) - ) - const extensions = [ ServerKit.configure({ image: { @@ -208,22 +191,7 @@ export async function start ( const wss = new WebSocketServer({ noServer: true, - perMessageDeflate: { - zlibDeflateOptions: { - chunkSize: 32 * 1024, - memLevel: 9, - level: 1 - }, - zlibInflateOptions: { - chunkSize: 32 * 1024, - memLevel: 9, - level: 1 - }, - // Below options specified as default values. - concurrencyLimit: 10, // Limits zlib concurrency for perf. - threshold: 1024 // Size (in bytes) below which messages - // should not be compressed if context takeover is disabled. - } + perMessageDeflate: false }) wss.on('connection', (incoming: WebSocket, request: IncomingMessage) => { diff --git a/server/core/src/adapter.ts b/server/core/src/adapter.ts index 4417a9d94f..bb52cd634b 100644 --- a/server/core/src/adapter.ts +++ b/server/core/src/adapter.ts @@ -66,14 +66,14 @@ export interface RawDBAdapter { workspace: WorkspaceId, domain: Domain, query: DocumentQuery, - options?: Omit, 'projection' | 'lookup'> + options?: Omit, 'projection' | 'lookup' | 'total'> ) => Promise> findStream: ( ctx: MeasureContext, workspace: WorkspaceId, domain: Domain, query: DocumentQuery, - options?: Omit, 'projection' | 'lookup'> + options?: Omit, 'projection' | 'lookup' | 'total'> ) => Promise> upload: (ctx: MeasureContext, workspace: WorkspaceId, domain: Domain, docs: T[]) => Promise update: ( diff --git a/server/middleware/src/spacePermissions.ts b/server/middleware/src/spacePermissions.ts index c180e2bbdb..16f8a07e6b 100644 --- a/server/middleware/src/spacePermissions.ts +++ b/server/middleware/src/spacePermissions.ts @@ -109,7 +109,17 @@ export class SpacePermissionsMiddleware extends BaseMiddleware implements Middle space, spaceType.targetClass ) as unknown as RolesAssignment - this.assignmentBySpace[space._id] = asMixin + + const allPossibleRoles = this.storage.modelDb.findAllSync(core.class.Role, {}) + const requiredValues: Record = {} + for (const role of allPossibleRoles) { + const v = asMixin[role._id] + if (v !== undefined) { + requiredValues[role._id] = asMixin[role._id] + } + } + + this.assignmentBySpace[space._id] = requiredValues await this.setPermissions(space._id, await this.getRoles(spaceType._id), asMixin) } @@ -193,7 +203,17 @@ export class SpacePermissionsMiddleware extends BaseMiddleware implements Middle // Note: currently the whole assignment is always included into the mixin update // so we can just rebuild the permissions const assignment: RolesAssignment = mixinDoc.attributes as RolesAssignment - this.assignmentBySpace[spaceId] = assignment + + const allPossibleRoles = this.storage.modelDb.findAllSync(core.class.Role, {}) + const requiredValues: Record = {} + for (const role of allPossibleRoles) { + const v = assignment[role._id] + if (v !== undefined) { + requiredValues[role._id] = assignment[role._id] + } + } + + this.assignmentBySpace[spaceId] = requiredValues this.permissionsBySpace[tx.objectId] = {} await this.setPermissions(spaceId, await this.getRoles(spaceType._id), assignment) diff --git a/server/mongo/package.json b/server/mongo/package.json index 88cc4b9db0..9081a4904c 100644 --- a/server/mongo/package.json +++ b/server/mongo/package.json @@ -37,7 +37,7 @@ "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", "@hcengineering/server-core": "^0.6.1", - "mongodb": "^6.3.0", - "bson": "^6.3.0" + "mongodb": "^6.8.0", + "bson": "^6.8.0" } } diff --git a/server/mongo/src/rawAdapter.ts b/server/mongo/src/rawAdapter.ts index a8e56dc4a7..b3e5e9adb2 100644 --- a/server/mongo/src/rawAdapter.ts +++ b/server/mongo/src/rawAdapter.ts @@ -42,7 +42,7 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter { workspace: WorkspaceId, domain: Domain, query: DocumentQuery, - options?: Omit, 'projection' | 'lookup'> + options?: Omit, 'projection' | 'lookup' | 'total'> ): Promise<{ cursor: FindCursor total: number @@ -54,7 +54,7 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter { checkKeys: false }) - let total: number = -1 + const total: number = -1 if (options != null) { if (options.sort !== undefined) { const sort = collectSort(options) @@ -63,9 +63,6 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter { } } if (options.limit !== undefined || typeof query._id === 'string') { - if (options.total === true) { - total = await coll.countDocuments(query) - } cursor = cursor.limit(options.limit ?? 1) } } @@ -78,9 +75,9 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter { workspace: WorkspaceId, domain: Domain, query: DocumentQuery, - options?: Omit, 'projection' | 'lookup'> + options?: Omit, 'projection' | 'lookup' | 'total'> ): Promise> { - let { cursor, total } = await ctx.with( + const { cursor, total } = await ctx.with( 'get-cursor', {}, async () => await getCursor(workspace, domain, query, options) @@ -92,9 +89,6 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter { ...query, ...options }) - if (options?.total === true && options?.limit === undefined) { - total = res.length - } return toFindResult(res, total) } catch (e) { console.error('error during executing cursor in findAll', cutObjectArray(query), options, e) @@ -106,7 +100,7 @@ export function createRawMongoDBAdapter (url: string): RawDBAdapter { workspace: WorkspaceId, domain: Domain, query: DocumentQuery, - options?: Omit, 'projection' | 'lookup'> + options?: Omit, 'projection' | 'lookup' | 'total'> ): Promise> { const { cursor } = await getCursor(workspace, domain, query, options) diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index 9d86f54441..003d5e5762 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -155,9 +155,11 @@ abstract class MongoAdapterBase implements DbAdapter { try { const existingIndexes = await this.collection(domain).indexes() for (const existingIndex of existingIndexes) { - const name: string = existingIndex.name - if (deletePattern.test(name) && !keepPattern.test(name)) { - await this.collection(domain).dropIndex(existingIndex.name) + if (existingIndex.name !== undefined) { + const name: string = existingIndex.name + if (deletePattern.test(name) && !keepPattern.test(name)) { + await this.collection(domain).dropIndex(name) + } } } } catch (err: any) { diff --git a/server/tool/package.json b/server/tool/package.json index 35fd50a4bb..ea85b0a050 100644 --- a/server/tool/package.json +++ b/server/tool/package.json @@ -38,7 +38,7 @@ "@types/jest": "^29.5.5" }, "dependencies": { - "mongodb": "^6.3.0", + "mongodb": "^6.8.0", "@hcengineering/platform": "^0.6.11", "@hcengineering/core": "^0.6.32", "@hcengineering/contact": "^0.6.24", diff --git a/server/ws/src/server.ts b/server/ws/src/server.ts index 6811fcc33a..004b2b0950 100644 --- a/server/ws/src/server.ts +++ b/server/ws/src/server.ts @@ -23,12 +23,12 @@ import core, { versionToString, withContext, type BaseWorkspaceInfo, + type Branding, + type BrandingMap, type MeasureContext, type Tx, type TxWorkspaceEvent, - type WorkspaceId, - type Branding, - type BrandingMap + type WorkspaceId } from '@hcengineering/core' import { unknownError, type Status } from '@hcengineering/platform' import { type HelloRequest, type HelloResponse, type Request, type Response } from '@hcengineering/rpc' @@ -260,8 +260,19 @@ class TSessionManager implements SessionManager { > { const wsString = toWorkspaceString(token.workspace, '@') - let workspaceInfo = - accountsUrl !== '' ? await this.getWorkspaceInfo(ctx, accountsUrl, rawToken) : this.wsFromToken(token) + let workspaceInfo: WorkspaceLoginInfo | undefined + for (let i = 0; i < 5; i++) { + try { + workspaceInfo = + accountsUrl !== '' ? await this.getWorkspaceInfo(ctx, accountsUrl, rawToken) : this.wsFromToken(token) + break + } catch (err: any) { + if (i === 4) { + throw err + } + await new Promise((resolve) => setTimeout(resolve, 10)) + } + } if (workspaceInfo?.creating === true && token.email !== systemAccountEmail) { // No access to workspace for token. @@ -302,7 +313,7 @@ class TSessionManager implements SessionManager { const workspaceName = workspaceInfo.workspaceName ?? workspaceInfo.workspaceUrl ?? workspaceInfo.workspaceId const branding = (workspaceInfo.branding !== undefined - ? Object.values(this.brandingMap).find((b) => b.key === workspaceInfo.branding) + ? Object.values(this.brandingMap).find((b) => b.key === (workspaceInfo as WorkspaceLoginInfo).branding) : null) ?? null if (workspace === undefined) { @@ -1007,7 +1018,7 @@ export function start ( opt.pipelineFactory, opt.port, opt.productId, - opt.enableCompression ?? true, + opt.enableCompression ?? false, opt.accountsUrl, opt.externalStorage ) diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index 01628ea015..94757c8a30 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -250,13 +250,13 @@ export function startHttpServer ( zlibDeflateOptions: { // See zlib defaults. chunkSize: 32 * 1024, - memLevel: 9, + memLevel: 1, level: 1 }, zlibInflateOptions: { chunkSize: 32 * 1024, level: 1, - memLevel: 9 + memLevel: 1 }, serverNoContextTakeover: true, clientNoContextTakeover: true, @@ -327,7 +327,7 @@ export function startHttpServer ( } catch (err: any) { Analytics.handleError(err) if (LOGGING_ENABLED) { - ctx.error('message error', err) + ctx.error('message error', { err }) } } }) diff --git a/server/ws/src/stats.ts b/server/ws/src/stats.ts index abda15364b..7c2ebe2d38 100644 --- a/server/ws/src/stats.ts +++ b/server/ws/src/stats.ts @@ -14,7 +14,7 @@ import { type SessionManager } from './types' */ export function getStatistics (ctx: MeasureContext, sessions: SessionManager, admin: boolean): any { const data: Record = { - metrics: metricsAggregate((ctx as any).metrics), + metrics: metricsAggregate((ctx as any).metrics, 50), statistics: { activeSessions: {} } @@ -40,8 +40,9 @@ export function getStatistics (ctx: MeasureContext, sessions: SessionManager, ad } } - data.statistics.memoryUsed = Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100 - data.statistics.memoryTotal = Math.round((process.memoryUsage().heapTotal / 1024 / 1024) * 100) / 100 + const memU = process.memoryUsage() + data.statistics.memoryUsed = Math.round(((memU.heapUsed + memU.rss) / 1024 / 1024) * 100) / 100 + data.statistics.memoryTotal = Math.round((memU.heapTotal / 1024 / 1024) * 100) / 100 data.statistics.cpuUsage = Math.round(os.loadavg()[0] * 100) / 100 data.statistics.freeMem = Math.round((os.freemem() / 1024 / 1024) * 100) / 100 data.statistics.totalMem = Math.round((os.totalmem() / 1024 / 1024) * 100) / 100