Merge remote-tracking branch 'origin/develop' into staging

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

.vscode/launch.json

@@ -129,7 +129,7 @@
         "MINIO_ACCESS_KEY": "minioadmin",
         "MINIO_SECRET_KEY": "minioadmin",
         "MINIO_ENDPOINT": "localhost",
-        "MODEL_VERSION": "v0.6.286"
+        "MODEL_VERSION": "v0.6.287"
         // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml",
         // "INIT_WORKSPACE": "onboarding",
       },

dev/tool/src/index.ts

@@ -1503,7 +1503,7 @@ export function devTool (
     .option('-w, --workspace <workspace>', 'Selected workspace only', '')
     .option('-c, --concurrency <concurrency>', 'Number of documents being processed concurrently', '10')
     .action(async (cmd: { workspace: string, concurrency: string }) => {
-      const { mongodbUri, dbUrl } = prepareTools()
+      const { mongodbUri, dbUrl, txes } = prepareTools()
       await withDatabase(mongodbUri, async (db, client) => {
         await withStorage(mongodbUri, async (adapter) => {
           const workspaces = await listWorkspacesPure(db)
@@ -1521,7 +1521,7 @@ export function devTool (
               workspaceUrl: workspace.workspaceUrl ?? ''
             }
 
-            const { pipeline } = await getServerPipeline(toolCtx, mongodbUri, dbUrl, wsUrl)
+            const { pipeline } = await getServerPipeline(toolCtx, txes, mongodbUri, dbUrl, wsUrl)
 
             await migrateMarkup(toolCtx, adapter, wsId, client, pipeline, parseInt(cmd.concurrency))
 

dev/tool/src/markup.ts

@@ -15,7 +15,7 @@ import core, {
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
 import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core'
-import { connect, fetchModel } from '@hcengineering/server-tool'
+import { connect } from '@hcengineering/server-tool'
 import { jsonToText, markupToYDoc } from '@hcengineering/text'
 import { type Db, type FindCursor, type MongoClient } from 'mongodb'
 
@@ -123,7 +123,7 @@ export async function migrateMarkup (
   pipeline: Pipeline,
   concurrency: number
 ): Promise<void> {
-  const { hierarchy } = await fetchModel(ctx, pipeline)
+  const hierarchy = pipeline.context.hierarchy
 
   const workspaceDb = client.db(workspaceId.name)
 

models/notification/src/index.ts

@@ -27,7 +27,6 @@ import {
   type Data,
   type Doc,
   type DocumentQuery,
-  type Domain,
   type IndexingConfiguration,
   type Markup,
   type Ref,
@@ -56,6 +55,9 @@ import view, { createAction, template } from '@hcengineering/model-view'
 import workbench from '@hcengineering/model-workbench'
 import {
   notificationId,
+  DOMAIN_USER_NOTIFY,
+  DOMAIN_NOTIFICATION,
+  DOMAIN_DOC_NOTIFY,
   type ActivityInboxNotification,
   type ActivityNotificationViewlet,
   type BaseNotificationType,
@@ -86,16 +88,10 @@ import { type AnyComponent, type Location } from '@hcengineering/ui/src/types'
 
 import notification from './plugin'
 
-export { notificationId } from '@hcengineering/notification'
+export { notificationId, DOMAIN_USER_NOTIFY, DOMAIN_NOTIFICATION, DOMAIN_DOC_NOTIFY } from '@hcengineering/notification'
 export { notificationOperation } from './migration'
 export { notification as default }
 
-export const DOMAIN_NOTIFICATION = 'notification' as Domain
-
-export const DOMAIN_DOC_NOTIFY = 'notification-dnc' as Domain
-
-export const DOMAIN_USER_NOTIFY = 'notification-user' as Domain
-
 @Model(notification.class.BrowserNotification, core.class.Doc, DOMAIN_USER_NOTIFY)
 export class TBrowserNotification extends TDoc implements BrowserNotification {
   senderId?: Ref<Account> | undefined

packages/ui/src/components/ListView.svelte

@@ -28,7 +28,7 @@
   export let lazy = false
   export let minHeight: string | null = null
   export let highlightIndex: number | undefined = undefined
-  const getKey: (index: number) => string = (index) => index.toString()
+  export let getKey: (index: number) => string = (index) => index.toString()
 
   const refs: HTMLElement[] = []
 

plugins/attachment-resources/src/components/AttachmentActions.svelte

@@ -23,7 +23,7 @@
     getPreviewAlignment,
     previewTypes
   } from '@hcengineering/presentation'
-  import { IconMoreH, IconOpen, Menu, Action as UIAction, closeTooltip, showPopup, tooltip } from '@hcengineering/ui'
+  import { IconMoreH, Menu, Action as UIAction, closeTooltip, showPopup, tooltip } from '@hcengineering/ui'
   import view, { Action } from '@hcengineering/view'
 
   import AttachmentAction from './AttachmentAction.svelte'
@@ -84,7 +84,7 @@
 
   const openAction: UIAction = {
     label: view.string.Open,
-    icon: IconOpen,
+    icon: view.icon.Open,
     action: async (props: any, evt: Event) => {
       showPreview(evt as MouseEvent)
     }
@@ -126,7 +126,7 @@
   {#if canPreview}
     <AttachmentAction
       label={view.string.Open}
-      icon={IconOpen}
+      icon={view.icon.Open}
       size="small"
       dataId="open-in-sidebar"
       action={showPreview}

plugins/attachment-resources/src/components/AttachmentPreview.svelte

@@ -51,7 +51,7 @@
       if (listProvider !== undefined) listProvider.updateFocus(value)
       const popupInfo = showPopup(
         FilePreviewPopup,
-        { file: value.file, name: value.name, contentType: value.type },
+        { file: value.file, name: value.name, contentType: value.type, metadata: value.metadata },
         value.type.startsWith('image/') ? 'centered' : 'float'
       )
       dispatch('open', popupInfo.id)

plugins/attachment-resources/src/components/Photos.svelte

@@ -92,7 +92,7 @@
     if (item !== undefined) {
       showPopup(
         FilePreviewPopup,
-        { file: item.file, name: item.name, contentType: item.type },
+        { file: item.file, name: item.name, contentType: item.type, metadata: item.metadata },
         item.type.startsWith('image/') ? 'centered' : 'float'
       )
     } else {

plugins/notification-resources/src/components/inbox/Inbox.svelte

@@ -171,7 +171,8 @@
 
     const [id, _class] = decodeObjectURI(loc?.loc.path[3] ?? '')
     const _id = await parseLinkId(linkProviders, id, _class)
-    const context = _id ? $contextByDocStore.get(_id) : undefined
+    const thread = loc?.loc.path[4] as Ref<ActivityMessage>
+    const context = $contextByDocStore.get(thread) ?? $contextByDocStore.get(_id)
 
     selectedContextId = context?._id
 
@@ -180,7 +181,6 @@
     }
 
     const selectedMessageId = loc?.loc.query?.message as Ref<ActivityMessage> | undefined
-    const thread = loc?.loc.path[4] as Ref<ActivityMessage> | undefined
 
     if (thread !== undefined) {
       const fn = await getResource(chunter.function.OpenThreadInSidebar)

plugins/notification-resources/src/components/inbox/InboxGroupedListView.svelte

@@ -93,6 +93,11 @@
   $: if (element != null) {
     element.focus()
   }
+
+  function getContextKey (index: number): string {
+    const contextId = displayData[index][0]
+    return contextId ?? index.toString()
+  }
 </script>
 
 <!-- svelte-ignore a11y-no-noninteractive-tabindex -->
@@ -108,6 +113,7 @@
     kind="full-size"
     colorsSchema="lumia"
     lazy={true}
+    getKey={getContextKey}
   >
     <svelte:fragment slot="item" let:item={itemIndex}>
       {@const contextId = displayData[itemIndex][0]}

plugins/notification/src/index.ts

@@ -20,6 +20,7 @@ import {
   Class,
   Doc,
   DocumentQuery,
+  type Domain,
   IdMap,
   Markup,
   Mixin,
@@ -42,6 +43,10 @@ import { Readable, Writable } from './types'
 
 export * from './types'
 
+export const DOMAIN_NOTIFICATION = 'notification' as Domain
+export const DOMAIN_DOC_NOTIFY = 'notification-dnc' as Domain
+export const DOMAIN_USER_NOTIFY = 'notification-user' as Domain
+
 /**
  * @public
  */

plugins/text-editor-resources/src/components/TextEditorToolbar.svelte

@@ -99,7 +99,7 @@
 
   function handleMouseDown (): void {
     function handleMouseMove (): void {
-      if (!editor.state.selection.empty) {
+      if (editor !== undefined && !editor.state.selection.empty) {
         selecting = true
         document.removeEventListener('mousemove', handleMouseMove)
       }
@@ -112,8 +112,10 @@
       document.removeEventListener('mouseup', handleMouseUp)
     }
 
-    document.addEventListener('mousemove', handleMouseMove)
-    document.addEventListener('mouseup', handleMouseUp)
+    if (editor !== undefined) {
+      document.addEventListener('mousemove', handleMouseMove)
+      document.addEventListener('mouseup', handleMouseUp)
+    }
   }
 
   onMount(() => {

pods/backup/package.json

@@ -57,6 +57,7 @@
     "dotenv": "~16.0.0",
     "@hcengineering/backup-service": "^0.6.0",
     "@hcengineering/analytics": "^0.6.0",
-    "@hcengineering/analytics-service": "^0.6.0"
+    "@hcengineering/analytics-service": "^0.6.0",
+    "@hcengineering/model-all": "^0.6.0"
   }
 }

pods/backup/src/index.ts

@@ -16,12 +16,19 @@
 import { Analytics } from '@hcengineering/analytics'
 import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service'
 import { startBackup } from '@hcengineering/backup-service'
-import { MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core'
+import { MeasureMetricsContext, metricsToString, newMetrics, type Tx } from '@hcengineering/core'
 import { type PipelineFactory } from '@hcengineering/server-core'
 import { createBackupPipeline, getConfig } from '@hcengineering/server-pipeline'
 import { writeFile } from 'fs/promises'
 import { join } from 'path'
 
+import builder from '@hcengineering/model-all'
+
+const enabled = (process.env.MODEL_ENABLED ?? '*').split(',').map((it) => it.trim())
+const disabled = (process.env.MODEL_DISABLED ?? '').split(',').map((it) => it.trim())
+
+const model = JSON.parse(JSON.stringify(builder(enabled, disabled).getTxes())) as Tx[]
+
 const metricsContext = new MeasureMetricsContext(
   'backup',
   {},
@@ -58,7 +65,7 @@ const onClose = (): void => {
 startBackup(
   metricsContext,
   (mongoUrl, storageAdapter) => {
-    const factory: PipelineFactory = createBackupPipeline(metricsContext, mongoUrl, {
+    const factory: PipelineFactory = createBackupPipeline(metricsContext, mongoUrl, model, {
       externalStorage: storageAdapter,
       usePassedCtx: true
     })

pods/server/package.json

@@ -72,6 +72,7 @@
     "@hcengineering/server-telegram": "^0.6.0",
     "@hcengineering/pod-telegram-bot": "^0.6.0",
     "@hcengineering/server-ai-bot": "^0.6.0",
-    "@hcengineering/server-ai-bot-resources": "^0.6.0"
+    "@hcengineering/server-ai-bot-resources": "^0.6.0",
+    "@hcengineering/model-all": "^0.6.0"
   }
 }

pods/server/src/server.ts

@@ -14,7 +14,7 @@
 // limitations under the License.
 //
 
-import { type Branding, type BrandingMap, type WorkspaceIdWithUrl } from '@hcengineering/core'
+import { type Branding, type BrandingMap, type Tx, type WorkspaceIdWithUrl } from '@hcengineering/core'
 import { buildStorageFromConfig, getMetricsContext } from '@hcengineering/server'
 
 import { ClientSession, startSessionManager, type ServerFactory, type Session } from '@hcengineering/server'
@@ -25,6 +25,13 @@ import { serverAiBotId } from '@hcengineering/server-ai-bot'
 import { createAIBotAdapter } from '@hcengineering/server-ai-bot-resources'
 import { createServerPipeline, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline'
 
+import builder from '@hcengineering/model-all'
+
+const enabled = (process.env.MODEL_ENABLED ?? '*').split(',').map((it) => it.trim())
+const disabled = (process.env.MODEL_DISABLED ?? '').split(',').map((it) => it.trim())
+
+const model = JSON.parse(JSON.stringify(builder(enabled, disabled).getTxes())) as Tx[]
+
 registerStringLoaders()
 
 /**
@@ -64,6 +71,7 @@ export function start (
   const pipelineFactory = createServerPipeline(
     metrics,
     dbUrls,
+    model,
     { ...opt, externalStorage, adapterSecurity: rawDbUrl !== undefined },
     {
       serviceAdapters: {

server-plugins/notification/src/index.ts

@@ -31,6 +31,7 @@ import type { TriggerControl, TriggerFunc } from '@hcengineering/server-core'
  * @public
  */
 export const serverNotificationId = 'server-notification' as Plugin
+export { DOMAIN_USER_NOTIFY, DOMAIN_NOTIFICATION, DOMAIN_DOC_NOTIFY } from '@hcengineering/notification'
 
 /**
  * @public

server/middleware/package.json

@@ -41,6 +41,7 @@
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/server-core": "^0.6.1",
     "@hcengineering/server-preference": "^0.6.0",
+    "@hcengineering/server-notification": "^0.6.1",
     "@hcengineering/query": "^0.6.12",
     "fast-equals": "^5.0.1"
   }

server/middleware/src/index.ts

@@ -33,3 +33,4 @@ export * from './spacePermissions'
 export * from './spaceSecurity'
 export * from './triggers'
 export * from './txPush'
+export * from './notifications'

server/middleware/src/model.ts

@@ -14,14 +14,22 @@
 //
 
 import core, {
+  type Doc,
   type LoadModelResponse,
   type MeasureContext,
   type Timestamp,
   type Tx,
+  type TxCUD,
   DOMAIN_TX
 } from '@hcengineering/core'
 import { PlatformError, unknownError } from '@hcengineering/platform'
-import type { Middleware, PipelineContext, TxAdapter, TxMiddlewareResult } from '@hcengineering/server-core'
+import type {
+  Middleware,
+  MiddlewareCreator,
+  PipelineContext,
+  TxAdapter,
+  TxMiddlewareResult
+} from '@hcengineering/server-core'
 import { BaseMiddleware } from '@hcengineering/server-core'
 import crypto from 'node:crypto'
 
@@ -34,20 +42,46 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware {
   lastHashResponse!: Promise<LoadModelResponse>
   model!: Tx[]
 
-  static async create (ctx: MeasureContext, context: PipelineContext, next?: Middleware): Promise<Middleware> {
-    const middleware = new ModelMiddleware(context, next)
+  constructor (
+    context: PipelineContext,
+    next: Middleware | undefined,
+    readonly systemTx: Tx[]
+  ) {
+    super(context, next)
+  }
+
+  static async doCreate (
+    ctx: MeasureContext,
+    context: PipelineContext,
+    next: Middleware | undefined,
+    systemTx: Tx[]
+  ): Promise<Middleware> {
+    const middleware = new ModelMiddleware(context, next, systemTx)
     await middleware.init(ctx)
     return middleware
   }
 
+  static create (tx: Tx[]): MiddlewareCreator {
+    return (ctx, context, next) => {
+      return this.doCreate(ctx, context, next, tx)
+    }
+  }
+
   async init (ctx: MeasureContext): Promise<void> {
     if (this.context.adapterManager == null) {
       throw new PlatformError(unknownError('Adapter manager should be configured'))
     }
     const txAdapter = this.context.adapterManager.getAdapter(DOMAIN_TX, true) as TxAdapter
 
+    const isUserTx = (it: Tx): boolean =>
+      it.modifiedBy !== core.account.System ||
+      (it as TxCUD<Doc>).objectClass === 'contact:class:Person' ||
+      (it as TxCUD<Doc>).objectClass === 'contact:class:PersonAccount'
+
     this.model = await ctx.with('get-model', {}, async (ctx) => {
-      const model = await ctx.with('fetch-model', {}, (ctx) => txAdapter.getModel(ctx))
+      const allUserTxes = await ctx.with('fetch-model', {}, (ctx) => txAdapter.getModel(ctx))
+      const userTxes = allUserTxes.filter((it) => isUserTx(it))
+      const model = this.systemTx.concat(userTxes)
       for (const tx of model) {
         try {
           this.context.hierarchy.tx(tx)
@@ -55,7 +89,7 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware {
           ctx.warn('failed to apply model transaction, skipping', { tx: JSON.stringify(tx), err })
         }
       }
-      this.context.modelDb.addTxes(ctx, model, false)
+      this.context.modelDb.addTxes(ctx, model, true)
       return model
     })
 

server/middleware/src/notifications.ts

@@ -0,0 +1,97 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import core, {
+  Doc,
+  MeasureContext,
+  Tx,
+  TxCUD,
+  TxProcessor,
+  systemAccountEmail,
+  type SessionData,
+  TxApplyIf
+} from '@hcengineering/core'
+import platform, { PlatformError, Severity, Status } from '@hcengineering/platform'
+import { BaseMiddleware, Middleware, TxMiddlewareResult, type PipelineContext } from '@hcengineering/server-core'
+import { DOMAIN_USER_NOTIFY, DOMAIN_NOTIFICATION, DOMAIN_DOC_NOTIFY } from '@hcengineering/server-notification'
+
+/**
+ * @public
+ */
+export class NotificationsMiddleware extends BaseMiddleware implements Middleware {
+  private readonly targetDomains = [DOMAIN_USER_NOTIFY, DOMAIN_NOTIFICATION, DOMAIN_DOC_NOTIFY]
+
+  private constructor (context: PipelineContext, next?: Middleware) {
+    super(context, next)
+  }
+
+  static async create (
+    ctx: MeasureContext,
+    context: PipelineContext,
+    next: Middleware | undefined
+  ): Promise<NotificationsMiddleware> {
+    return new NotificationsMiddleware(context, next)
+  }
+
+  isTargetDomain (tx: Tx): boolean {
+    if (TxProcessor.isExtendsCUD(tx._class)) {
+      const txCUD = tx as TxCUD<Doc>
+      const domain = this.context.hierarchy.getDomain(txCUD.objectClass)
+      return this.targetDomains.includes(domain)
+    }
+    return false
+  }
+
+  processTx (ctx: MeasureContext<SessionData>, tx: Tx): void {
+    let target: string[] | undefined
+    if (this.isTargetDomain(tx)) {
+      const account = ctx.contextData.account._id
+      if (account !== tx.modifiedBy && account !== core.account.System) {
+        throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {}))
+      }
+      const modifiedByAccount = ctx.contextData.getAccount(tx.modifiedBy)
+      target = [ctx.contextData.userEmail, systemAccountEmail]
+      if (modifiedByAccount !== undefined && !target.includes(modifiedByAccount.email)) {
+        target.push(modifiedByAccount.email)
+      }
+      ctx.contextData.broadcast.targets['checkDomain' + account] = (tx) => {
+        if (this.isTargetDomain(tx)) {
+          return target
+        }
+      }
+    }
+  }
+
+  tx (ctx: MeasureContext<SessionData>, txes: Tx[]): Promise<TxMiddlewareResult> {
+    for (const tx of txes) {
+      if (this.context.hierarchy.isDerived(tx._class, core.class.TxApplyIf)) {
+        for (const ttx of (tx as TxApplyIf).txes) {
+          this.processTx(ctx, ttx)
+        }
+      } else {
+        this.processTx(ctx, tx)
+      }
+    }
+
+    return this.provideTx(ctx, txes)
+  }
+
+  isAvailable (ctx: MeasureContext<SessionData>, doc: Doc): boolean {
+    const domain = this.context.hierarchy.getDomain(doc._class)
+    if (!this.targetDomains.includes(domain)) return true
+    const account = ctx.contextData.account._id
+    return doc.createdBy === account || account === core.account.System
+  }
+}

server/mongo/src/storage.ts

@@ -1174,7 +1174,9 @@ abstract class MongoAdapterBase implements DbAdapter {
 
   async clean (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
     await ctx.with('clean', {}, async () => {
-      await this.db.collection<Doc>(domain).deleteMany({ _id: { $in: docs } })
+      if (docs.length > 0) {
+        await this.db.collection<Doc>(domain).deleteMany({ _id: { $in: docs } })
+      }
     })
   }
 }
@@ -1693,7 +1695,7 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter {
       )
     }
     model.forEach((tx) => (tx.modifiedBy === core.account.System && !isPersonAccount(tx) ? systemTx : userTx).push(tx))
-    return systemTx.concat(userTx)
+    return this.stripHash(systemTx.concat(userTx)) as Tx[]
   }
 }
 

server/mongo/src/utils.ts

@@ -169,6 +169,7 @@ export class DBCollectionHelper implements DomainHelperOperations {
   }
 
   async init (domain?: Domain): Promise<void> {
+    // Check and create DB if missin
     if (domain === undefined) {
       // Init existing collecfions
       for (const c of (await this.db.listCollections({}, { nameOnly: true }).toArray()).map((it) => it.name)) {

server/server-pipeline/src/pipeline.ts

@@ -10,6 +10,7 @@ import {
   ModelDb,
   type Branding,
   type MeasureContext,
+  type Tx,
   type WorkspaceIdWithUrl
 } from '@hcengineering/core'
 import { createElasticAdapter, createElasticBackupDataAdapter } from '@hcengineering/elastic'
@@ -33,7 +34,8 @@ import {
   SpacePermissionsMiddleware,
   SpaceSecurityMiddleware,
   TriggersMiddleware,
-  TxMiddleware
+  TxMiddleware,
+  NotificationsMiddleware
 } from '@hcengineering/middleware'
 import { createMongoAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
 import { createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
@@ -99,6 +101,7 @@ export function getTxAdapterFactory (
 export function createServerPipeline (
   metrics: MeasureContext,
   dbUrls: string,
+  model: Tx[],
   opt: {
     fullTextUrl: string
     rekoniUrl: string
@@ -121,6 +124,7 @@ export function createServerPipeline (
       LookupMiddleware.create,
       ModifiedMiddleware.create,
       PrivateMiddleware.create,
+      NotificationsMiddleware.create,
       (ctx: MeasureContext, context: PipelineContext, next?: Middleware) =>
         SpaceSecurityMiddleware.create(opt.adapterSecurity ?? false, ctx, context, next),
       SpacePermissionsMiddleware.create,
@@ -137,7 +141,7 @@ export function createServerPipeline (
       DomainFindMiddleware.create,
       DomainTxMiddleware.create,
       DBAdapterInitMiddleware.create,
-      ModelMiddleware.create,
+      ModelMiddleware.create(model),
       DBAdapterMiddleware.create(conf), // Configure DB adapters
       BroadcastMiddleware.create(broadcast)
     ]
@@ -162,6 +166,7 @@ export function createServerPipeline (
 export function createBackupPipeline (
   metrics: MeasureContext,
   dbUrls: string,
+  systemTx: Tx[],
   opt: {
     usePassedCtx?: boolean
     adapterSecurity?: boolean
@@ -206,7 +211,7 @@ export function createBackupPipeline (
       ContextNameMiddleware.create,
       DomainFindMiddleware.create,
       DBAdapterInitMiddleware.create,
-      ModelMiddleware.create,
+      ModelMiddleware.create(systemTx),
       DBAdapterMiddleware.create(conf)
     ]
 
@@ -225,6 +230,7 @@ export function createBackupPipeline (
 
 export async function getServerPipeline (
   ctx: MeasureContext,
+  model: Tx[],
   mongodbUri: string,
   dbUrl: string | undefined,
   wsUrl: WorkspaceIdWithUrl
@@ -240,6 +246,7 @@ export async function getServerPipeline (
   const pipelineFactory = createServerPipeline(
     ctx,
     dbUrls,
+    model,
     {
       externalStorage: storageAdapter,
       fullTextUrl: 'http://localhost:9200',

server/tool/src/index.ts

@@ -36,15 +36,16 @@ import core, {
   WorkspaceId,
   WorkspaceIdWithUrl,
   type Doc,
+  type Ref,
   type TxCUD
 } from '@hcengineering/core'
 import { consoleModelLogger, MigrateOperation, ModelLogger, tryMigrate } from '@hcengineering/model'
 import {
   AggregatorStorageAdapter,
-  DbAdapter,
   DomainIndexHelperImpl,
   Pipeline,
-  StorageAdapter
+  StorageAdapter,
+  type DbAdapter
 } from '@hcengineering/server-core'
 import { connect } from './connect'
 import { InitScript, WorkspaceInitializer } from './initializer'
@@ -141,9 +142,16 @@ export async function initModel (
       logger.log('transactions deleted.', { workspaceId: workspaceId.name })
     }
 
-    logger.log('creating model...', workspaceId)
-    await adapter.upload(ctx, DOMAIN_TX, txes)
-    logger.log('model transactions inserted.', { count: txes.length })
+    logger.log('creating database...', workspaceId)
+    await adapter.upload(ctx, DOMAIN_TX, [
+      {
+        _class: core.class.Tx,
+        _id: 'first-tx' as Ref<Doc>,
+        modifiedBy: core.account.System,
+        modifiedOn: Date.now(),
+        space: core.space.DerivedTx
+      }
+    ])
 
     await progress(30)
 
@@ -264,11 +272,14 @@ export async function upgradeModel (
     throw Error('Model txes must target only core.space.Model')
   }
 
-  const prevModel = await fetchModel(ctx, pipeline)
+  const newModelRes = await ctx.with('load-model', {}, (ctx) => pipeline.loadModel(ctx, 0))
+  const newModel = Array.isArray(newModelRes) ? newModelRes : newModelRes.transactions
+
+  const { hierarchy, modelDb, model } = await buildModel(ctx, newModel)
   const { migrateClient: preMigrateClient } = await prepareMigrationClient(
     pipeline,
-    prevModel.hierarchy,
-    prevModel.modelDb,
+    hierarchy,
+    modelDb,
     logger,
     storageAdapter,
     workspaceId
@@ -305,36 +316,9 @@ export async function upgradeModel (
     }
     logger.log('removing model...', { workspaceId: workspaceId.name })
     await progress(10)
-    const toRemove = await pipeline.findAll(ctx, core.class.Tx, {
-      objectSpace: core.space.Model,
-      modifiedBy: core.account.System,
-      objectClass: { $nin: [contact.class.PersonAccount, 'contact:class:EmployeeAccount'] }
-    })
-    await pipeline.context.lowLevelStorage.clean(
-      ctx,
-      DOMAIN_TX,
-      toRemove.map((p) => p._id)
-    )
-    logger.log('transactions deleted.', { workspaceId: workspaceId.name, count: toRemove.length })
-    logger.log('creating model...', { workspaceId: workspaceId.name })
-    await pipeline.context.lowLevelStorage.upload(ctx, DOMAIN_TX, txes)
-
     logger.log('model transactions inserted.', { workspaceId: workspaceId.name, count: txes.length })
     await progress(20)
   }
-  const newModel = [
-    ...txes,
-    ...Array.from(
-      prevModel.model.filter(
-        (it) =>
-          it.modifiedBy !== core.account.System ||
-          (it as TxCUD<Doc>).objectClass === contact.class.Person ||
-          (it as TxCUD<Doc>).objectClass === 'contact:class:PersonAccount'
-      )
-    )
-  ]
-
-  const { hierarchy, modelDb, model } = await fetchModel(ctx, pipeline, newModel)
   const { migrateClient, migrateState } = await prepareMigrationClient(
     pipeline,
     hierarchy,
@@ -383,6 +367,21 @@ export async function upgradeModel (
       {
         state: 'indexes-v4',
         func: upgradeIndexes
+      },
+      {
+        state: 'delete-model',
+        func: async (client) => {
+          const model = await client.find<Tx>(DOMAIN_TX, { objectSpace: core.space.Model })
+
+          // Ignore Employee accounts.
+          const isUserTx = (it: Tx): boolean =>
+            it.modifiedBy !== core.account.System ||
+            (it as TxCUD<Doc>).objectClass === 'contact:class:Person' ||
+            (it as TxCUD<Doc>).objectClass === 'contact:class:PersonAccount'
+
+          const toDelete = model.filter((it) => !isUserTx(it)).map((it) => it._id)
+          await client.deleteMany(DOMAIN_TX, { _id: { $in: toDelete } })
+        }
       }
     ])
   })
@@ -463,26 +462,20 @@ async function prepareMigrationClient (
   return { migrateClient, migrateState }
 }
 
-export async function fetchModel (
+export async function buildModel (
   ctx: MeasureContext,
-  pipeline: Pipeline,
-  model?: Tx[]
+  model: Tx[]
 ): Promise<{ hierarchy: Hierarchy, modelDb: ModelDb, model: Tx[] }> {
   const hierarchy = new Hierarchy()
   const modelDb = new ModelDb(hierarchy)
 
-  if (model === undefined) {
-    const res = await ctx.with('load-model', {}, (ctx) => pipeline.loadModel(ctx, 0))
-    model = Array.isArray(res) ? res : res.transactions
-  }
-
   ctx.withSync('build local model', {}, () => {
     for (const tx of model ?? []) {
       try {
         hierarchy.tx(tx)
       } catch (err: any) {}
     }
-    modelDb.addTxes(ctx, model as Tx[], false)
+    modelDb.addTxes(ctx, model, false)
   })
   return { hierarchy, modelDb, model: model ?? [] }
 }

server/workspace-service/src/ws-operations.ts

@@ -163,6 +163,7 @@ export async function createWorkspace (
       const factory: PipelineFactory = createServerPipeline(
         ctx,
         dbUrls,
+        txes,
         {
           externalStorage: storageAdapter,
           fullTextUrl: 'http://localhost:9200',
@@ -288,7 +289,7 @@ export async function upgradeWorkspace (
   let pipeline: Pipeline | undefined
   let storageAdapter: StorageAdapter | undefined
   try {
-    ;({ pipeline, storageAdapter } = await getServerPipeline(ctx, mongodbUri, dbUrl, wsUrl))
+    ;({ pipeline, storageAdapter } = await getServerPipeline(ctx, txes, mongodbUri, dbUrl, wsUrl))
     const contextData = new SessionDataImpl(
       systemAccountEmail,
       'backup',