From f3db427f1d420a2d3631c89eec4b2433f32dda63 Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@users.noreply.github.com>
Date: Fri, 6 Oct 2023 14:20:34 +0700
Subject: [PATCH] Client model persistence (#3796)
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
packages/core/src/client.ts | 46 +++++++++++++++++++++------
plugins/client-resources/src/index.ts | 19 ++++++++++-
server/core/src/indexer/indexer.ts | 28 +++++++++++++++-
3 files changed, 82 insertions(+), 11 deletions(-)
diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts
index a8601caa03..566b2b4223 100644
--- a/packages/core/src/client.ts
+++ b/packages/core/src/client.ts
@@ -158,13 +158,22 @@ class ClientImpl implements AccountClient, BackupClient {
}
}
+/**
+ * @public
+ */
+export interface TxPersistenceStore {
+ load: () => Promise<Tx[]>
+ store: (tx: Tx[]) => Promise<void>
+}
+
/**
* @public
*/
export async function createClient (
connect: (txHandler: TxHandler) => Promise<ClientConnection>,
// If set will build model with only allowed plugins.
- allowedPlugins?: Plugin[]
+ allowedPlugins?: Plugin[],
+ txPersistence?: TxPersistenceStore
): Promise<AccountClient> {
let client: ClientImpl | null = null
@@ -193,7 +202,7 @@ export async function createClient (
const conn = await connect(txHandler)
- lastTxTime = await loadModel(conn, lastTxTime, allowedPlugins, configs, hierarchy, model)
+ lastTxTime = await loadModel(conn, lastTxTime, allowedPlugins, configs, hierarchy, model, false, txPersistence)
txBuffer = txBuffer.filter((tx) => tx.space !== core.space.Model || tx.modifiedOn > lastTxTime)
@@ -255,11 +264,18 @@ async function loadModel (
configs: Map<Ref<PluginConfiguration>, PluginConfiguration>,
hierarchy: Hierarchy,
model: ModelDb,
- reload = false
+ reload = false,
+ persistence?: TxPersistenceStore
): Promise<Timestamp> {
const t = Date.now()
- let atxes = []
+ let ltxes: Tx[] = []
+ if (lastTxTime === 0 && persistence !== undefined) {
+ ltxes = await persistence.load()
+ lastTxTime = getLastTxTime(ltxes)
+ }
+
+ let atxes: Tx[] = []
try {
atxes = await conn.loadModel(lastTxTime)
} catch (err: any) {
@@ -274,6 +290,12 @@ async function loadModel (
return -1
}
+ if (atxes.length < modelTransactionThreshold) {
+ atxes = ltxes.concat(atxes)
+ }
+
+ await persistence?.store(atxes)
+
let systemTx: Tx[] = []
const userTx: Tx[] = []
console.log('find' + (lastTxTime >= 0 ? 'full model' : 'model diff'), atxes.length, Date.now() - t)
@@ -302,11 +324,7 @@ async function loadModel (
const txes = systemTx.concat(userTx)
- for (const tx of txes) {
- if (tx.modifiedOn > lastTxTime) {
- lastTxTime = tx.modifiedOn
- }
- }
+ lastTxTime = getLastTxTime(txes)
for (const tx of txes) {
try {
@@ -325,6 +343,16 @@ async function loadModel (
return lastTxTime
}
+function getLastTxTime (txes: Tx[]): number {
+ let lastTxTime = 0
+ for (const tx of txes) {
+ if (tx.modifiedOn > lastTxTime) {
+ lastTxTime = tx.modifiedOn
+ }
+ }
+ return lastTxTime
+}
+
function fillConfiguration (systemTx: Tx[], configs: Map<Ref<PluginConfiguration>, PluginConfiguration>): void {
for (const t of systemTx) {
if (t._class === core.class.TxCreateDoc) {
diff --git a/plugins/client-resources/src/index.ts b/plugins/client-resources/src/index.ts
index 496b712ec6..424c7393fc 100644
--- a/plugins/client-resources/src/index.ts
+++ b/plugins/client-resources/src/index.ts
@@ -60,7 +60,24 @@ export default async () => {
return connect(url.href, upgradeHandler, onUpgrade, onUnauthorized, onConnect)
},
- filterModel ? [...getPlugins(), ...(getMetadata(clientPlugin.metadata.ExtraPlugins) ?? [])] : undefined
+ filterModel ? [...getPlugins(), ...(getMetadata(clientPlugin.metadata.ExtraPlugins) ?? [])] : undefined,
+ {
+ load: async () => {
+ if (typeof localStorage !== 'undefined') {
+ const dta = localStorage.getItem('stored_model_' + token) ?? null
+ if (dta === null) {
+ return []
+ }
+ return JSON.parse(dta)
+ }
+ return []
+ },
+ store: async (txes) => {
+ if (typeof localStorage !== 'undefined') {
+ localStorage.setItem('stored_model_' + token, JSON.stringify(txes))
+ }
+ }
+ }
)
// Check if we had dev hook for client.
client = hookClient(client)
diff --git a/server/core/src/indexer/indexer.ts b/server/core/src/indexer/indexer.ts
index d70f241558..d74c0cc47e 100644
--- a/server/core/src/indexer/indexer.ts
+++ b/server/core/src/indexer/indexer.ts
@@ -30,6 +30,7 @@ import core, {
WorkspaceId,
_getOperator,
setObjectValue,
+ toFindResult,
versionToString
} from '@hcengineering/core'
import { DbAdapter } from '../adapter'
@@ -371,7 +372,7 @@ export class FullTextIndexPipeline implements FullTextPipeline {
.filter((it) => it[1] > 3)
.map((it) => it[0])
- const result = await this.metrics.with(
+ let result = await this.metrics.with(
'get-to-index',
{},
async () =>
@@ -390,6 +391,31 @@ export class FullTextIndexPipeline implements FullTextPipeline {
}
)
)
+ const toRemove: DocIndexState[] = []
+ // Check and remove missing class documents.
+ result = toFindResult(
+ result.filter((doc) => {
+ const _class = this.model.findObject(doc.objectClass)
+ if (_class === undefined) {
+ // no _class present, remove doc
+ toRemove.push(doc)
+ return false
+ }
+ return true
+ }),
+ result.total
+ )
+
+ if (toRemove.length > 0) {
+ try {
+ await this.storage.clean(
+ DOMAIN_DOC_INDEX_STATE,
+ toRemove.map((it) => it._id)
+ )
+ } catch (err: any) {
+ // QuotaExceededError, ignore
+ }
+ }
if (result.length > 0) {
console.log(