Cacheable multiple query (#2009)

Signed-off-by: Denis Bykhov <80476319+BykhovDenis@users.noreply.github.com>
This commit is contained in:
Denis Bykhov 2022-06-06 11:42:51 +06:00 committed by GitHub
parent a66c26e56a
commit dbc25d6ae9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 197 additions and 93 deletions

View File

@ -234,6 +234,7 @@ specifiers:
eslint-plugin-import: ^2.25.3
eslint-plugin-node: ^11.1.0
eslint-plugin-promise: ^5.1.1
eslint-plugin-svelte3: ^4.0.0
express: ^4.17.1
express-fileupload: ^1.2.1
faker: ~5.5.3
@ -517,6 +518,7 @@ dependencies:
eslint-plugin-import: 2.26.0_c21022bc9feaeb7b200d3d631eeae46c
eslint-plugin-node: 11.1.0_eslint@7.32.0
eslint-plugin-promise: 5.2.0_eslint@7.32.0
eslint-plugin-svelte3: 4.0.0_eslint@7.32.0+svelte@3.48.0
express: 4.18.1
express-fileupload: 1.4.0
faker: 5.5.3
@ -12021,7 +12023,7 @@ packages:
dev: false
file:projects/model-tags.tgz_typescript@4.7.2:
resolution: {integrity: sha512-GqDd2wBGXXzJlWiXxn/I2YB6qbwyFLt8o8OR7xdpaDEAF6YNbym+7u+Ptq0O9kW9msT3hLvmS/vk5HUq0HMnsg==, tarball: file:projects/model-tags.tgz}
resolution: {integrity: sha512-W7hggW0TyIrPDN1nGGcNUQHbYoy++o9apSGHND0nFmFHSmtRug4w613wGqH9dolQzjEb5vNoPC1g40JiebUBGg==, tarball: file:projects/model-tags.tgz}
id: file:projects/model-tags.tgz
name: '@rush-temp/model-tags'
version: 0.0.0
@ -12667,7 +12669,7 @@ packages:
dev: false
file:projects/query.tgz:
resolution: {integrity: sha512-sFTZR0opxCdNZTeIIBcJJcwiWDD/fffdFb3W78Ht0SGzOjhCK/tZm/Pwdq5Uxd9Nh+CqYHzQnjVf0CtAY9oZmg==, tarball: file:projects/query.tgz}
resolution: {integrity: sha512-1czqFbKUfNjFTLv9E/ZZpKcv8K/OphFY+K0N8s+KJDRAwWu7vQ2TX5pIdb8AfpZIjea3s98bmTEPUhpIo31p1A==, tarball: file:projects/query.tgz}
name: '@rush-temp/query'
version: 0.0.0
dependencies:
@ -12680,8 +12682,10 @@ packages:
eslint-plugin-import: 2.26.0_c21022bc9feaeb7b200d3d631eeae46c
eslint-plugin-node: 11.1.0_eslint@7.32.0
eslint-plugin-promise: 5.2.0_eslint@7.32.0
fast-equals: 2.0.4
prettier: 2.6.2
simplytyped: 3.3.0_typescript@4.7.2
svelte: 3.48.0
typescript: 4.7.2
transitivePeerDependencies:
- eslint-import-resolver-typescript
@ -12775,7 +12779,7 @@ packages:
dev: false
file:projects/rekoni.tgz_svelte@3.48.0:
resolution: {integrity: sha512-wurMqQ+zqRTVEuYvmfTh6QDsv1h0UyUeIXsXEmbk5YrQXNUE6z/sCAWvq3DkrvsMol4DArVtrL+TsZNV1vkDCw==, tarball: file:projects/rekoni.tgz}
resolution: {integrity: sha512-WX6fbaayz7h8GYFe8hhsTqsv9Un38RR8RRiALVaiicNtAzjZsk9HJuRJWz/fkTdEm6Tqs43/16phEmTVcAmhAQ==, tarball: file:projects/rekoni.tgz}
id: file:projects/rekoni.tgz
name: '@rush-temp/rekoni'
version: 0.0.0
@ -13763,7 +13767,7 @@ packages:
dev: false
file:projects/tags.tgz:
resolution: {integrity: sha512-EShaylPW6dK736SqcQhPxvEUfnrdUNFcx91jInmgAnx0/IHpGAay9votdRx9+S+fPo5jwo4hh6PQR1b0Z0DIRA==, tarball: file:projects/tags.tgz}
resolution: {integrity: sha512-sh+ncMhE1ARIndBYIKL1pCbPOjeRZjyzXH1ps28xr9YEyrItgFov0A+bUtr56AqoAtm2XNK6fTqkor8t1eEdDw==, tarball: file:projects/tags.tgz}
name: '@rush-temp/tags'
version: 0.0.0
dependencies:

View File

@ -29,6 +29,7 @@
},
"dependencies": {
"@anticrm/platform": "~0.6.6",
"@anticrm/core": "~0.6.16"
"@anticrm/core": "~0.6.16",
"fast-equals": "^2.0.3"
}
}

View File

@ -469,14 +469,13 @@ describe('query', () => {
(result) => {
const comment = result[0]
if (comment !== undefined) {
if (attempt > 0) {
if (attempt++ > 0) {
expect((comment.$lookup?.attachedTo as WithLookup<AttachedComment>)?.$lookup?.space?._id).toEqual(
futureSpace._id
)
resolve(null)
} else {
expect((comment.$lookup?.attachedTo as WithLookup<AttachedComment>)?.$lookup?.space).toBeUndefined()
attempt++
}
}
},
@ -616,12 +615,13 @@ describe('query', () => {
message: 'child'
}
)
let attempt = 0
let attempt = -1
const pp = new Promise((resolve) => {
liveQuery.query<AttachedComment>(
test.class.TestComment,
{ _id: childComment },
(result) => {
attempt++
const comment = result[0]
if (comment !== undefined) {
if (attempt > 0) {
@ -631,7 +631,6 @@ describe('query', () => {
expect(
((comment.$lookup?.attachedTo as WithLookup<AttachedComment>)?.$lookup?.space as Doc)?._id
).toEqual(futureSpace)
attempt++
}
}
},
@ -659,7 +658,7 @@ describe('query', () => {
message: 'test'
}
)
let attempt = 0
let attempt = -1
const childLength = 3
const childs: Ref<AttachedComment>[] = []
for (let index = 0; index < childLength; index++) {
@ -681,10 +680,10 @@ describe('query', () => {
test.class.TestComment,
{ _id: parentComment },
(result) => {
attempt++
const comment = result[0]
if (comment !== undefined) {
expect((comment.$lookup as any)?.comments).toHaveLength(childLength - attempt)
attempt++
}
if (attempt === childLength) {
resolve(null)
@ -755,7 +754,7 @@ describe('query', () => {
it('lookup nested query update doc', async () => {
const { liveQuery, factory } = await getClient()
let attempt = 0
let attempt = -1
const futureSpace = await factory.createDoc(core.class.Space, core.space.Model, {
name: '0',
description: '',
@ -788,6 +787,7 @@ describe('query', () => {
test.class.TestComment,
{ _id: childComment },
(result) => {
attempt++
const comment = result[0]
if (comment !== undefined) {
expect(
@ -796,8 +796,6 @@ describe('query', () => {
}
if (attempt > 0) {
resolve(null)
} else {
attempt++
}
},
{ lookup: { attachedTo: [test.class.TestComment, { space: core.class.Space }] } }
@ -823,7 +821,7 @@ describe('query', () => {
message: 'test'
}
)
let attempt = 0
let attempt = -1
const childComment = await factory.addCollection(
test.class.TestComment,
spaces[0]._id,
@ -839,14 +837,13 @@ describe('query', () => {
test.class.TestComment,
{ _id: parentComment },
(result) => {
attempt++
const comment = result[0]
if (comment !== undefined) {
expect(((comment.$lookup as any)?.comments[0] as AttachedComment).message).toEqual(attempt.toString())
}
if (attempt > 0) {
resolve(null)
} else {
attempt++
}
},
{ lookup: { _id: { comments: test.class.TestComment } } }

View File

@ -44,6 +44,11 @@ import core, {
WithLookup,
toFindResult
} from '@anticrm/core'
import { deepEqual } from 'fast-equals'
const CACHE_SIZE = 20
type Callback = (result: FindResult<Doc>) => void
interface Query {
_class: Ref<Class<Doc>>
@ -51,7 +56,7 @@ interface Query {
result: Doc[] | Promise<Doc[]>
options?: FindOptions<Doc>
total: number
callback: (result: FindResult<Doc>) => void
callbacks: Callback[]
}
/**
@ -59,7 +64,8 @@ interface Query {
*/
export class LiveQuery extends TxProcessor implements Client {
private readonly client: Client
private readonly queries: Query[] = []
private readonly queries: Map<Ref<Class<Doc>>, Query[]> = new Map<Ref<Class<Doc>>, Query[]>()
private readonly queue: Query[] = []
constructor (client: Client) {
super()
@ -114,12 +120,59 @@ export class LiveQuery extends TxProcessor implements Client {
return await this.client.findOne(_class, query, options)
}
query<T extends Doc>(
private findQuery<T extends Doc>(
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
options?: FindOptions<T>
): Query | undefined {
const queries = this.queries.get(_class)
if (queries === undefined) return
for (const q of queries) {
if (!deepEqual(query, q.query) || !deepEqual(options, q.options)) continue
return q
}
}
private removeFromQueue (q: Query): void {
if (q.callbacks.length === 0) {
const queueIndex = this.queue.indexOf(q)
if (queueIndex !== -1) {
this.queue.splice(queueIndex, 1)
}
}
}
private pushCallback (q: Query, callback: (result: Doc[]) => void): void {
q.callbacks.push(callback)
setTimeout(async () => {
if (q !== undefined) {
await this.callback(q)
}
}, 0)
}
private getQuery<T extends Doc>(
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
callback: (result: Doc[]) => void,
options?: FindOptions<T>
): Query | undefined {
const current = this.findQuery(_class, query, options)
if (current !== undefined) {
this.removeFromQueue(current)
this.pushCallback(current, callback)
return current
}
}
private createQuery<T extends Doc>(
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
callback: (result: FindResult<T>) => void,
options?: FindOptions<T>
): () => void {
): Query {
const queries = this.queries.get(_class) ?? []
const result = this.client.findAll(_class, query, options)
const q: Query = {
_class,
@ -127,60 +180,97 @@ export class LiveQuery extends TxProcessor implements Client {
result,
total: 0,
options: options as FindOptions<Doc>,
callback: callback as (result: Doc[]) => void
callbacks: [callback as (result: Doc[]) => void]
}
this.queries.push(q)
queries.push(q)
result
.then((result) => {
.then(async (result) => {
q.total = result.total
q.callback(result)
await this.callback(q)
})
.catch((err) => {
console.log('failed to update Live Query: ', err)
})
this.queries.set(_class, queries)
if (this.queue.length > CACHE_SIZE) {
this.remove()
}
return q
}
private remove (): void {
const q = this.queue.shift()
if (q === undefined) return
const queries = this.queries.get(q._class)
queries?.splice(queries.indexOf(q), 1)
if (queries?.length === 0) {
this.queries.delete(q._class)
}
}
query<T extends Doc>(
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
callback: (result: FindResult<T>) => void,
options?: FindOptions<T>
): () => void {
const q =
this.getQuery(_class, query, callback as (result: Doc[]) => void, options) ??
this.createQuery(_class, query, callback, options)
return () => {
q.callback = () => {}
this.queries.splice(this.queries.indexOf(q), 1)
const index = q.callbacks.indexOf(callback as (result: Doc[]) => void)
if (index !== -1) {
q.callbacks.splice(index, 1)
}
if (q.callbacks.length === 0) {
this.queue.push(q)
}
}
}
protected override async txPutBag (tx: TxPutBag<any>): Promise<TxResult> {
for (const q of this.queries) {
if (q.result instanceof Promise) {
q.result = await q.result
}
const updatedDoc = q.result.find((p) => p._id === tx.objectId)
if (updatedDoc !== undefined) {
const doc = updatedDoc as any
let bag = doc[tx.bag]
if (bag === undefined) {
doc[tx.bag] = bag = {}
for (const queries of this.queries.values()) {
for (const q of queries) {
if (q.result instanceof Promise) {
q.result = await q.result
}
const updatedDoc = q.result.find((p) => p._id === tx.objectId)
if (updatedDoc !== undefined) {
const doc = updatedDoc as any
let bag = doc[tx.bag]
if (bag === undefined) {
doc[tx.bag] = bag = {}
}
bag[tx.key] = tx.value
await this.updatedDocCallback(updatedDoc, q)
}
bag[tx.key] = tx.value
await this.updatedDocCallback(updatedDoc, q)
}
}
return {}
}
protected override async txMixin (tx: TxMixin<Doc, Doc>): Promise<TxResult> {
for (const q of this.queries) {
if (this.client.getHierarchy().isDerived(q._class, core.class.Tx)) {
// handle add since Txes are immutable
await this.handleDocAdd(q, tx)
continue
}
if (q.result instanceof Promise) {
q.result = await q.result
}
let updatedDoc = q.result.find((p) => p._id === tx.objectId)
if (updatedDoc !== undefined) {
// Create or apply mixin value
updatedDoc = TxProcessor.updateMixin4Doc(updatedDoc, tx.mixin, tx.attributes)
await this.updatedDocCallback(updatedDoc, q)
} else {
if (this.getHierarchy().isDerived(tx.mixin, q._class)) {
const hierarchy = this.client.getHierarchy()
for (const queries of this.queries) {
const isTx = hierarchy.isDerived(queries[0], core.class.Tx)
const isMixin = hierarchy.isDerived(tx.mixin, queries[0])
for (const q of queries[1]) {
if (isTx) {
// handle add since Txes are immutable
await this.handleDocAdd(q, tx)
continue
}
if (q.result instanceof Promise) {
q.result = await q.result
}
let updatedDoc = q.result.find((p) => p._id === tx.objectId)
if (updatedDoc !== undefined) {
// Create or apply mixin value
updatedDoc = TxProcessor.updateMixin4Doc(updatedDoc, tx.mixin, tx.attributes)
await this.updatedDocCallback(updatedDoc, q)
} else if (isMixin) {
// Mixin potentially added to object we doesn't have in out results
const doc = await this.findOne(q._class, { _id: tx.objectId }, q.options)
if (doc !== undefined) {
@ -193,42 +283,48 @@ export class LiveQuery extends TxProcessor implements Client {
}
protected async txCollectionCUD (tx: TxCollectionCUD<Doc, AttachedDoc>): Promise<TxResult> {
for (const q of this.queries) {
if (this.client.getHierarchy().isDerived(q._class, core.class.Tx)) {
// handle add since Txes are immutable
await this.handleDocAdd(q, tx)
continue
}
if (tx.tx._class === core.class.TxCreateDoc) {
const createTx = tx.tx as TxCreateDoc<AttachedDoc>
const d: TxCreateDoc<AttachedDoc> = {
...createTx,
attributes: {
...createTx.attributes,
attachedTo: tx.objectId,
attachedToClass: tx.objectClass,
collection: tx.collection
}
for (const queries of this.queries) {
const isTx = this.client.getHierarchy().isDerived(queries[0], core.class.Tx)
for (const q of queries[1]) {
if (isTx) {
// handle add since Txes are immutable
await this.handleDocAdd(q, tx)
continue
}
if (tx.tx._class === core.class.TxCreateDoc) {
const createTx = tx.tx as TxCreateDoc<AttachedDoc>
const d: TxCreateDoc<AttachedDoc> = {
...createTx,
attributes: {
...createTx.attributes,
attachedTo: tx.objectId,
attachedToClass: tx.objectClass,
collection: tx.collection
}
}
await this.handleDocAdd(q, TxProcessor.createDoc2Doc(d))
} else if (tx.tx._class === core.class.TxUpdateDoc) {
await this.handleDocUpdate(q, tx.tx as unknown as TxUpdateDoc<Doc>)
} else if (tx.tx._class === core.class.TxRemoveDoc) {
await this.handleDocRemove(q, tx.tx as unknown as TxRemoveDoc<Doc>)
}
await this.handleDocAdd(q, TxProcessor.createDoc2Doc(d))
} else if (tx.tx._class === core.class.TxUpdateDoc) {
await this.handleDocUpdate(q, tx.tx as unknown as TxUpdateDoc<Doc>)
} else if (tx.tx._class === core.class.TxRemoveDoc) {
await this.handleDocRemove(q, tx.tx as unknown as TxRemoveDoc<Doc>)
}
}
return {}
}
protected async txUpdateDoc (tx: TxUpdateDoc<Doc>): Promise<TxResult> {
for (const q of this.queries) {
if (this.client.getHierarchy().isDerived(q._class, core.class.Tx)) {
// handle add since Txes are immutable
await this.handleDocAdd(q, tx)
continue
for (const queries of this.queries) {
const isTx = this.client.getHierarchy().isDerived(queries[0], core.class.Tx)
for (const q of queries[1]) {
if (isTx) {
// handle add since Txes are immutable
await this.handleDocAdd(q, tx)
continue
}
await this.handleDocUpdate(q, tx)
}
await this.handleDocUpdate(q, tx)
}
return {}
}
@ -414,9 +510,11 @@ export class LiveQuery extends TxProcessor implements Client {
protected async txCreateDoc (tx: TxCreateDoc<Doc>): Promise<TxResult> {
const docTx = TxProcessor.createDoc2Doc(tx)
for (const q of this.queries) {
const doc = this.client.getHierarchy().isDerived(q._class, core.class.Tx) ? tx : docTx
await this.handleDocAdd(q, doc)
for (const queries of this.queries) {
const doc = this.client.getHierarchy().isDerived(queries[0], core.class.Tx) ? tx : docTx
for (const q of queries[1]) {
await this.handleDocAdd(q, doc)
}
}
return {}
}
@ -466,7 +564,8 @@ export class LiveQuery extends TxProcessor implements Client {
q.result = await q.result
}
const clone = this.clone(q.result)
q.callback(toFindResult(clone, q.total))
const result = toFindResult(clone, q.total)
q.callbacks.forEach((callback) => callback(result))
}
private async handleDocAddLookup (q: Query, doc: Doc): Promise<void> {
@ -514,13 +613,16 @@ export class LiveQuery extends TxProcessor implements Client {
}
protected async txRemoveDoc (tx: TxRemoveDoc<Doc>): Promise<TxResult> {
for (const q of this.queries) {
if (this.client.getHierarchy().isDerived(q._class, core.class.Tx)) {
// handle add since Txes are immutable
await this.handleDocAdd(q, tx)
continue
for (const queries of this.queries) {
const isTx = this.client.getHierarchy().isDerived(queries[0], core.class.Tx)
for (const q of queries[1]) {
if (isTx) {
// handle add since Txes are immutable
await this.handleDocAdd(q, tx)
continue
}
await this.handleDocRemove(q, tx)
}
await this.handleDocRemove(q, tx)
}
return {}
}