From dbc25d6ae937beb3fbfa34f4f026babba080dfda Mon Sep 17 00:00:00 2001 From: Denis Bykhov <80476319+BykhovDenis@users.noreply.github.com> Date: Mon, 6 Jun 2022 11:42:51 +0600 Subject: [PATCH] Cacheable multiple query (#2009) Signed-off-by: Denis Bykhov <80476319+BykhovDenis@users.noreply.github.com> --- common/config/rush/pnpm-lock.yaml | 12 +- packages/query/package.json | 3 +- packages/query/src/__tests__/query.test.ts | 21 +- packages/query/src/index.ts | 254 +++++++++++++++------ 4 files changed, 197 insertions(+), 93 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index b0cd09d1a9..5a5e7f9ba9 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -234,6 +234,7 @@ specifiers: eslint-plugin-import: ^2.25.3 eslint-plugin-node: ^11.1.0 eslint-plugin-promise: ^5.1.1 + eslint-plugin-svelte3: ^4.0.0 express: ^4.17.1 express-fileupload: ^1.2.1 faker: ~5.5.3 @@ -517,6 +518,7 @@ dependencies: eslint-plugin-import: 2.26.0_c21022bc9feaeb7b200d3d631eeae46c eslint-plugin-node: 11.1.0_eslint@7.32.0 eslint-plugin-promise: 5.2.0_eslint@7.32.0 + eslint-plugin-svelte3: 4.0.0_eslint@7.32.0+svelte@3.48.0 express: 4.18.1 express-fileupload: 1.4.0 faker: 5.5.3 @@ -12021,7 +12023,7 @@ packages: dev: false file:projects/model-tags.tgz_typescript@4.7.2: - resolution: {integrity: sha512-GqDd2wBGXXzJlWiXxn/I2YB6qbwyFLt8o8OR7xdpaDEAF6YNbym+7u+Ptq0O9kW9msT3hLvmS/vk5HUq0HMnsg==, tarball: file:projects/model-tags.tgz} + resolution: {integrity: sha512-W7hggW0TyIrPDN1nGGcNUQHbYoy++o9apSGHND0nFmFHSmtRug4w613wGqH9dolQzjEb5vNoPC1g40JiebUBGg==, tarball: file:projects/model-tags.tgz} id: file:projects/model-tags.tgz name: '@rush-temp/model-tags' version: 0.0.0 @@ -12667,7 +12669,7 @@ packages: dev: false file:projects/query.tgz: - resolution: {integrity: sha512-sFTZR0opxCdNZTeIIBcJJcwiWDD/fffdFb3W78Ht0SGzOjhCK/tZm/Pwdq5Uxd9Nh+CqYHzQnjVf0CtAY9oZmg==, tarball: file:projects/query.tgz} + resolution: {integrity: sha512-1czqFbKUfNjFTLv9E/ZZpKcv8K/OphFY+K0N8s+KJDRAwWu7vQ2TX5pIdb8AfpZIjea3s98bmTEPUhpIo31p1A==, tarball: file:projects/query.tgz} name: '@rush-temp/query' version: 0.0.0 dependencies: @@ -12680,8 +12682,10 @@ packages: eslint-plugin-import: 2.26.0_c21022bc9feaeb7b200d3d631eeae46c eslint-plugin-node: 11.1.0_eslint@7.32.0 eslint-plugin-promise: 5.2.0_eslint@7.32.0 + fast-equals: 2.0.4 prettier: 2.6.2 simplytyped: 3.3.0_typescript@4.7.2 + svelte: 3.48.0 typescript: 4.7.2 transitivePeerDependencies: - eslint-import-resolver-typescript @@ -12775,7 +12779,7 @@ packages: dev: false file:projects/rekoni.tgz_svelte@3.48.0: - resolution: {integrity: sha512-wurMqQ+zqRTVEuYvmfTh6QDsv1h0UyUeIXsXEmbk5YrQXNUE6z/sCAWvq3DkrvsMol4DArVtrL+TsZNV1vkDCw==, tarball: file:projects/rekoni.tgz} + resolution: {integrity: sha512-WX6fbaayz7h8GYFe8hhsTqsv9Un38RR8RRiALVaiicNtAzjZsk9HJuRJWz/fkTdEm6Tqs43/16phEmTVcAmhAQ==, tarball: file:projects/rekoni.tgz} id: file:projects/rekoni.tgz name: '@rush-temp/rekoni' version: 0.0.0 @@ -13763,7 +13767,7 @@ packages: dev: false file:projects/tags.tgz: - resolution: {integrity: sha512-EShaylPW6dK736SqcQhPxvEUfnrdUNFcx91jInmgAnx0/IHpGAay9votdRx9+S+fPo5jwo4hh6PQR1b0Z0DIRA==, tarball: file:projects/tags.tgz} + resolution: {integrity: sha512-sh+ncMhE1ARIndBYIKL1pCbPOjeRZjyzXH1ps28xr9YEyrItgFov0A+bUtr56AqoAtm2XNK6fTqkor8t1eEdDw==, tarball: file:projects/tags.tgz} name: '@rush-temp/tags' version: 0.0.0 dependencies: diff --git a/packages/query/package.json b/packages/query/package.json index f267dd9840..7494b9a07b 100644 --- a/packages/query/package.json +++ b/packages/query/package.json @@ -29,6 +29,7 @@ }, "dependencies": { "@anticrm/platform": "~0.6.6", - "@anticrm/core": "~0.6.16" + "@anticrm/core": "~0.6.16", + "fast-equals": "^2.0.3" } } diff --git a/packages/query/src/__tests__/query.test.ts b/packages/query/src/__tests__/query.test.ts index 59dd789fc8..f156019ffc 100644 --- a/packages/query/src/__tests__/query.test.ts +++ b/packages/query/src/__tests__/query.test.ts @@ -469,14 +469,13 @@ describe('query', () => { (result) => { const comment = result[0] if (comment !== undefined) { - if (attempt > 0) { + if (attempt++ > 0) { expect((comment.$lookup?.attachedTo as WithLookup)?.$lookup?.space?._id).toEqual( futureSpace._id ) resolve(null) } else { expect((comment.$lookup?.attachedTo as WithLookup)?.$lookup?.space).toBeUndefined() - attempt++ } } }, @@ -616,12 +615,13 @@ describe('query', () => { message: 'child' } ) - let attempt = 0 + let attempt = -1 const pp = new Promise((resolve) => { liveQuery.query( test.class.TestComment, { _id: childComment }, (result) => { + attempt++ const comment = result[0] if (comment !== undefined) { if (attempt > 0) { @@ -631,7 +631,6 @@ describe('query', () => { expect( ((comment.$lookup?.attachedTo as WithLookup)?.$lookup?.space as Doc)?._id ).toEqual(futureSpace) - attempt++ } } }, @@ -659,7 +658,7 @@ describe('query', () => { message: 'test' } ) - let attempt = 0 + let attempt = -1 const childLength = 3 const childs: Ref[] = [] for (let index = 0; index < childLength; index++) { @@ -681,10 +680,10 @@ describe('query', () => { test.class.TestComment, { _id: parentComment }, (result) => { + attempt++ const comment = result[0] if (comment !== undefined) { expect((comment.$lookup as any)?.comments).toHaveLength(childLength - attempt) - attempt++ } if (attempt === childLength) { resolve(null) @@ -755,7 +754,7 @@ describe('query', () => { it('lookup nested query update doc', async () => { const { liveQuery, factory } = await getClient() - let attempt = 0 + let attempt = -1 const futureSpace = await factory.createDoc(core.class.Space, core.space.Model, { name: '0', description: '', @@ -788,6 +787,7 @@ describe('query', () => { test.class.TestComment, { _id: childComment }, (result) => { + attempt++ const comment = result[0] if (comment !== undefined) { expect( @@ -796,8 +796,6 @@ describe('query', () => { } if (attempt > 0) { resolve(null) - } else { - attempt++ } }, { lookup: { attachedTo: [test.class.TestComment, { space: core.class.Space }] } } @@ -823,7 +821,7 @@ describe('query', () => { message: 'test' } ) - let attempt = 0 + let attempt = -1 const childComment = await factory.addCollection( test.class.TestComment, spaces[0]._id, @@ -839,14 +837,13 @@ describe('query', () => { test.class.TestComment, { _id: parentComment }, (result) => { + attempt++ const comment = result[0] if (comment !== undefined) { expect(((comment.$lookup as any)?.comments[0] as AttachedComment).message).toEqual(attempt.toString()) } if (attempt > 0) { resolve(null) - } else { - attempt++ } }, { lookup: { _id: { comments: test.class.TestComment } } } diff --git a/packages/query/src/index.ts b/packages/query/src/index.ts index de18486bc1..b3e502cb70 100644 --- a/packages/query/src/index.ts +++ b/packages/query/src/index.ts @@ -44,6 +44,11 @@ import core, { WithLookup, toFindResult } from '@anticrm/core' +import { deepEqual } from 'fast-equals' + +const CACHE_SIZE = 20 + +type Callback = (result: FindResult) => void interface Query { _class: Ref> @@ -51,7 +56,7 @@ interface Query { result: Doc[] | Promise options?: FindOptions total: number - callback: (result: FindResult) => void + callbacks: Callback[] } /** @@ -59,7 +64,8 @@ interface Query { */ export class LiveQuery extends TxProcessor implements Client { private readonly client: Client - private readonly queries: Query[] = [] + private readonly queries: Map>, Query[]> = new Map>, Query[]>() + private readonly queue: Query[] = [] constructor (client: Client) { super() @@ -114,12 +120,59 @@ export class LiveQuery extends TxProcessor implements Client { return await this.client.findOne(_class, query, options) } - query( + private findQuery( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Query | undefined { + const queries = this.queries.get(_class) + if (queries === undefined) return + for (const q of queries) { + if (!deepEqual(query, q.query) || !deepEqual(options, q.options)) continue + return q + } + } + + private removeFromQueue (q: Query): void { + if (q.callbacks.length === 0) { + const queueIndex = this.queue.indexOf(q) + if (queueIndex !== -1) { + this.queue.splice(queueIndex, 1) + } + } + } + + private pushCallback (q: Query, callback: (result: Doc[]) => void): void { + q.callbacks.push(callback) + setTimeout(async () => { + if (q !== undefined) { + await this.callback(q) + } + }, 0) + } + + private getQuery( + _class: Ref>, + query: DocumentQuery, + callback: (result: Doc[]) => void, + options?: FindOptions + ): Query | undefined { + const current = this.findQuery(_class, query, options) + if (current !== undefined) { + this.removeFromQueue(current) + this.pushCallback(current, callback) + + return current + } + } + + private createQuery( _class: Ref>, query: DocumentQuery, callback: (result: FindResult) => void, options?: FindOptions - ): () => void { + ): Query { + const queries = this.queries.get(_class) ?? [] const result = this.client.findAll(_class, query, options) const q: Query = { _class, @@ -127,60 +180,97 @@ export class LiveQuery extends TxProcessor implements Client { result, total: 0, options: options as FindOptions, - callback: callback as (result: Doc[]) => void + callbacks: [callback as (result: Doc[]) => void] } - this.queries.push(q) + queries.push(q) result - .then((result) => { + .then(async (result) => { q.total = result.total - q.callback(result) + await this.callback(q) }) .catch((err) => { console.log('failed to update Live Query: ', err) }) + this.queries.set(_class, queries) + if (this.queue.length > CACHE_SIZE) { + this.remove() + } + return q + } + + private remove (): void { + const q = this.queue.shift() + if (q === undefined) return + const queries = this.queries.get(q._class) + queries?.splice(queries.indexOf(q), 1) + if (queries?.length === 0) { + this.queries.delete(q._class) + } + } + + query( + _class: Ref>, + query: DocumentQuery, + callback: (result: FindResult) => void, + options?: FindOptions + ): () => void { + const q = + this.getQuery(_class, query, callback as (result: Doc[]) => void, options) ?? + this.createQuery(_class, query, callback, options) + return () => { - q.callback = () => {} - this.queries.splice(this.queries.indexOf(q), 1) + const index = q.callbacks.indexOf(callback as (result: Doc[]) => void) + if (index !== -1) { + q.callbacks.splice(index, 1) + } + if (q.callbacks.length === 0) { + this.queue.push(q) + } } } protected override async txPutBag (tx: TxPutBag): Promise { - for (const q of this.queries) { - if (q.result instanceof Promise) { - q.result = await q.result - } - const updatedDoc = q.result.find((p) => p._id === tx.objectId) - if (updatedDoc !== undefined) { - const doc = updatedDoc as any - let bag = doc[tx.bag] - if (bag === undefined) { - doc[tx.bag] = bag = {} + for (const queries of this.queries.values()) { + for (const q of queries) { + if (q.result instanceof Promise) { + q.result = await q.result + } + const updatedDoc = q.result.find((p) => p._id === tx.objectId) + if (updatedDoc !== undefined) { + const doc = updatedDoc as any + let bag = doc[tx.bag] + if (bag === undefined) { + doc[tx.bag] = bag = {} + } + bag[tx.key] = tx.value + await this.updatedDocCallback(updatedDoc, q) } - bag[tx.key] = tx.value - await this.updatedDocCallback(updatedDoc, q) } } return {} } protected override async txMixin (tx: TxMixin): Promise { - for (const q of this.queries) { - if (this.client.getHierarchy().isDerived(q._class, core.class.Tx)) { - // handle add since Txes are immutable - await this.handleDocAdd(q, tx) - continue - } - if (q.result instanceof Promise) { - q.result = await q.result - } - let updatedDoc = q.result.find((p) => p._id === tx.objectId) - if (updatedDoc !== undefined) { - // Create or apply mixin value - updatedDoc = TxProcessor.updateMixin4Doc(updatedDoc, tx.mixin, tx.attributes) - await this.updatedDocCallback(updatedDoc, q) - } else { - if (this.getHierarchy().isDerived(tx.mixin, q._class)) { + const hierarchy = this.client.getHierarchy() + for (const queries of this.queries) { + const isTx = hierarchy.isDerived(queries[0], core.class.Tx) + const isMixin = hierarchy.isDerived(tx.mixin, queries[0]) + for (const q of queries[1]) { + if (isTx) { + // handle add since Txes are immutable + await this.handleDocAdd(q, tx) + continue + } + if (q.result instanceof Promise) { + q.result = await q.result + } + let updatedDoc = q.result.find((p) => p._id === tx.objectId) + if (updatedDoc !== undefined) { + // Create or apply mixin value + updatedDoc = TxProcessor.updateMixin4Doc(updatedDoc, tx.mixin, tx.attributes) + await this.updatedDocCallback(updatedDoc, q) + } else if (isMixin) { // Mixin potentially added to object we doesn't have in out results const doc = await this.findOne(q._class, { _id: tx.objectId }, q.options) if (doc !== undefined) { @@ -193,42 +283,48 @@ export class LiveQuery extends TxProcessor implements Client { } protected async txCollectionCUD (tx: TxCollectionCUD): Promise { - for (const q of this.queries) { - if (this.client.getHierarchy().isDerived(q._class, core.class.Tx)) { - // handle add since Txes are immutable - await this.handleDocAdd(q, tx) - continue - } - - if (tx.tx._class === core.class.TxCreateDoc) { - const createTx = tx.tx as TxCreateDoc - const d: TxCreateDoc = { - ...createTx, - attributes: { - ...createTx.attributes, - attachedTo: tx.objectId, - attachedToClass: tx.objectClass, - collection: tx.collection - } + for (const queries of this.queries) { + const isTx = this.client.getHierarchy().isDerived(queries[0], core.class.Tx) + for (const q of queries[1]) { + if (isTx) { + // handle add since Txes are immutable + await this.handleDocAdd(q, tx) + continue + } + + if (tx.tx._class === core.class.TxCreateDoc) { + const createTx = tx.tx as TxCreateDoc + const d: TxCreateDoc = { + ...createTx, + attributes: { + ...createTx.attributes, + attachedTo: tx.objectId, + attachedToClass: tx.objectClass, + collection: tx.collection + } + } + await this.handleDocAdd(q, TxProcessor.createDoc2Doc(d)) + } else if (tx.tx._class === core.class.TxUpdateDoc) { + await this.handleDocUpdate(q, tx.tx as unknown as TxUpdateDoc) + } else if (tx.tx._class === core.class.TxRemoveDoc) { + await this.handleDocRemove(q, tx.tx as unknown as TxRemoveDoc) } - await this.handleDocAdd(q, TxProcessor.createDoc2Doc(d)) - } else if (tx.tx._class === core.class.TxUpdateDoc) { - await this.handleDocUpdate(q, tx.tx as unknown as TxUpdateDoc) - } else if (tx.tx._class === core.class.TxRemoveDoc) { - await this.handleDocRemove(q, tx.tx as unknown as TxRemoveDoc) } } return {} } protected async txUpdateDoc (tx: TxUpdateDoc): Promise { - for (const q of this.queries) { - if (this.client.getHierarchy().isDerived(q._class, core.class.Tx)) { - // handle add since Txes are immutable - await this.handleDocAdd(q, tx) - continue + for (const queries of this.queries) { + const isTx = this.client.getHierarchy().isDerived(queries[0], core.class.Tx) + for (const q of queries[1]) { + if (isTx) { + // handle add since Txes are immutable + await this.handleDocAdd(q, tx) + continue + } + await this.handleDocUpdate(q, tx) } - await this.handleDocUpdate(q, tx) } return {} } @@ -414,9 +510,11 @@ export class LiveQuery extends TxProcessor implements Client { protected async txCreateDoc (tx: TxCreateDoc): Promise { const docTx = TxProcessor.createDoc2Doc(tx) - for (const q of this.queries) { - const doc = this.client.getHierarchy().isDerived(q._class, core.class.Tx) ? tx : docTx - await this.handleDocAdd(q, doc) + for (const queries of this.queries) { + const doc = this.client.getHierarchy().isDerived(queries[0], core.class.Tx) ? tx : docTx + for (const q of queries[1]) { + await this.handleDocAdd(q, doc) + } } return {} } @@ -466,7 +564,8 @@ export class LiveQuery extends TxProcessor implements Client { q.result = await q.result } const clone = this.clone(q.result) - q.callback(toFindResult(clone, q.total)) + const result = toFindResult(clone, q.total) + q.callbacks.forEach((callback) => callback(result)) } private async handleDocAddLookup (q: Query, doc: Doc): Promise { @@ -514,13 +613,16 @@ export class LiveQuery extends TxProcessor implements Client { } protected async txRemoveDoc (tx: TxRemoveDoc): Promise { - for (const q of this.queries) { - if (this.client.getHierarchy().isDerived(q._class, core.class.Tx)) { - // handle add since Txes are immutable - await this.handleDocAdd(q, tx) - continue + for (const queries of this.queries) { + const isTx = this.client.getHierarchy().isDerived(queries[0], core.class.Tx) + for (const q of queries[1]) { + if (isTx) { + // handle add since Txes are immutable + await this.handleDocAdd(q, tx) + continue + } + await this.handleDocRemove(q, tx) } - await this.handleDocRemove(q, tx) } return {} }