From 8e840c7d3c949af8e2a8a3f1126883bc3b54ed28 Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@users.noreply.github.com>
Date: Sun, 13 Oct 2024 23:59:38 +0700
Subject: [PATCH 01/12] UBERF-8455: Fix admin users (#6909)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
 models/tracker/src/viewlets.ts                |  8 +++++-
 .../projects/ProjectPresenter.svelte          | 13 ++++++++--
 .../src/components/navigator/NavLink.svelte   |  3 ++-
 .../src/components/Navigator.svelte           | 16 +++++++-----
 server/middleware/src/spaceSecurity.ts        | 25 ++++++-------------
 server/middleware/src/utils.ts                |  4 +--
 6 files changed, 40 insertions(+), 29 deletions(-)

diff --git a/models/tracker/src/viewlets.ts b/models/tracker/src/viewlets.ts
index ac55bde0ee..ee25b63197 100644
--- a/models/tracker/src/viewlets.ts
+++ b/models/tracker/src/viewlets.ts
@@ -528,7 +528,13 @@ export function defineViewlets (builder: Builder): void {
         hiddenKeys: ['identifier', 'name', 'description']
       },
       config: [
-        '',
+        {
+          key: '',
+          presenter: tracker.component.ProjectPresenter,
+          props: {
+            openIssues: true
+          }
+        },
         'members',
         {
           key: 'defaultAssignee',
diff --git a/plugins/tracker-resources/src/components/projects/ProjectPresenter.svelte b/plugins/tracker-resources/src/components/projects/ProjectPresenter.svelte
index a68ced6268..b694ea84ec 100644
--- a/plugins/tracker-resources/src/components/projects/ProjectPresenter.svelte
+++ b/plugins/tracker-resources/src/components/projects/ProjectPresenter.svelte
@@ -13,7 +13,7 @@
 // limitations under the License.
 -->
 <script lang="ts">
-  import presentation from '@hcengineering/presentation'
+  import presentation, { isAdminUser } from '@hcengineering/presentation'
   import { Project } from '@hcengineering/tracker'
   import {
     Icon,
@@ -24,12 +24,15 @@
     themeStore
   } from '@hcengineering/ui'
   import view from '@hcengineering/view'
+  import { NavLink } from '@hcengineering/view-resources'
   import tracker from '../../plugin'
+  import { getCurrentAccount } from '@hcengineering/core'
 
   export let value: Project | undefined
   export let inline: boolean = false
   export let accent: boolean = false
   export let colorInherit: boolean = false
+  export let openIssues: boolean
 </script>
 
 {#if value}
@@ -49,7 +52,13 @@
       />
     </div>
     <span class="label no-underline nowrap" class:fs-bold={accent}>
-      {value.name}
+      {#if openIssues && (isAdminUser() || value.members.includes(getCurrentAccount()._id))}
+        <NavLink space={value._id} special={'issues'} noUnderline={false}>
+          {value.name}
+        </NavLink>
+      {:else}
+        {value.name}
+      {/if}
       {#if value.archived}
         <Label label={presentation.string.Archived} />
       {/if}
diff --git a/plugins/view-resources/src/components/navigator/NavLink.svelte b/plugins/view-resources/src/components/navigator/NavLink.svelte
index 62f047a630..39fd9864a7 100644
--- a/plugins/view-resources/src/components/navigator/NavLink.svelte
+++ b/plugins/view-resources/src/components/navigator/NavLink.svelte
@@ -22,6 +22,7 @@
   export let disabled = false
   export let shrink: number | undefined = undefined
   export let restoreLastLocation = false
+  export let noUnderline = true
 
   $: loc = createLocation($location, app, space, special)
 
@@ -76,7 +77,7 @@
 {#if disabled}
   <slot />
 {:else}
-  <a class="noUnderline noBold" style:flex-shrink={shrink} {href} on:click={clickHandler}>
+  <a class:noUnderline class="noBold" style:flex-shrink={shrink} {href} on:click={clickHandler}>
     <slot />
   </a>
 {/if}
diff --git a/plugins/workbench-resources/src/components/Navigator.svelte b/plugins/workbench-resources/src/components/Navigator.svelte
index 8536d5aeb7..10b72261af 100644
--- a/plugins/workbench-resources/src/components/Navigator.svelte
+++ b/plugins/workbench-resources/src/components/Navigator.svelte
@@ -16,7 +16,7 @@
   import core, { Doc, Ref, SortingOrder, Space, getCurrentAccount, hasAccountRole } from '@hcengineering/core'
   import { getResource } from '@hcengineering/platform'
   import preference, { SpacePreference } from '@hcengineering/preference'
-  import { createQuery, getClient } from '@hcengineering/presentation'
+  import { createQuery, getClient, isAdminUser } from '@hcengineering/presentation'
   import { Scroller, NavItem } from '@hcengineering/ui'
   import { NavLink } from '@hcengineering/view-resources'
   import type { Application, NavigatorModel, SpecialNavModel } from '@hcengineering/workbench'
@@ -40,6 +40,8 @@
   let starred: Space[] = []
   let shownSpaces: Space[] = []
 
+  const adminUser = isAdminUser()
+
   $: if (model) {
     const classes = Array.from(new Set(getSpecialSpaceClass(model).flatMap((c) => hierarchy.getDescendants(c)))).filter(
       (it) => !hierarchy.isMixin(it)
@@ -47,10 +49,12 @@
     if (classes.length > 0) {
       query.query(
         classes.length === 1 ? classes[0] : core.class.Space,
-        {
-          ...(classes.length === 1 ? {} : { _class: { $in: classes } }),
-          members: getCurrentAccount()._id
-        },
+        !adminUser
+          ? {
+              ...(classes.length === 1 ? {} : { _class: { $in: classes } }),
+              members: getCurrentAccount()._id
+            }
+          : { ...(classes.length === 1 ? {} : { _class: { $in: classes } }) },
         (result) => {
           spaces = result
         },
@@ -187,7 +191,7 @@
       />
     {/if}
 
-    {#each model.spaces as m, i (m.label)}
+    {#each model.spaces as m (m.label)}
       <SpacesNav
         spaces={shownSpaces.filter((it) => hierarchy.isDerived(it._class, m.spaceClass))}
         {currentSpace}
diff --git a/server/middleware/src/spaceSecurity.ts b/server/middleware/src/spaceSecurity.ts
index 04e447e047..a7e76eeba1 100644
--- a/server/middleware/src/spaceSecurity.ts
+++ b/server/middleware/src/spaceSecurity.ts
@@ -503,17 +503,14 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
     const isSpace = this.context.hierarchy.isDerived(_class, core.class.Space)
     const field = this.getKey(domain)
 
-    if (
-      ctx.contextData.admin === true &&
-      this.context.hierarchy.isDerived(_class, core.class.Space) &&
-      (newQuery as DocumentQuery<Space>).members !== undefined
-    ) {
-      delete (newQuery as any).members
-    }
-
     let clientFilterSpaces: Set<Ref<Space>> | undefined
 
-    if (!this.skipFindCheck && !isSystem(account) && account.role !== AccountRole.DocGuest && domain !== DOMAIN_MODEL) {
+    if (
+      !this.skipFindCheck &&
+      !isSystem(account, ctx) &&
+      account.role !== AccountRole.DocGuest &&
+      domain !== DOMAIN_MODEL
+    ) {
       if (!isOwner(account, ctx) || !isSpace) {
         if (query[field] !== undefined) {
           const res = await this.mergeQuery(ctx, account, query[field], domain, isSpace)
@@ -566,12 +563,6 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
         }
       }
     }
-    if (ctx.contextData.admin === true && this.context.hierarchy.isDerived(_class, core.class.Space)) {
-      // We need to add amin to all spaces.
-      for (const d of findResult) {
-        ;(d as unknown as Space).members = [...((d as unknown as Space).members ?? []), ctx.contextData.account._id]
-      }
-    }
     return findResult
   }
 
@@ -583,7 +574,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
     await this.init(ctx)
     const newQuery = { ...query }
     const account = ctx.contextData.account
-    if (!isSystem(account)) {
+    if (!isSystem(account, ctx)) {
       const allSpaces = this.getAllAllowedSpaces(account, true)
       if (query.classes !== undefined) {
         const res = new Set<Ref<Space>>()
@@ -610,7 +601,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
 
   async isUnavailable (ctx: MeasureContext<SessionData>, space: Ref<Space>): Promise<boolean> {
     const account = ctx.contextData.account
-    if (isSystem(account)) return false
+    if (isSystem(account, ctx)) return false
     return !this.getAllAllowedSpaces(account, true).includes(space)
   }
 
diff --git a/server/middleware/src/utils.ts b/server/middleware/src/utils.ts
index 95e188f5de..1ab7e9a17f 100644
--- a/server/middleware/src/utils.ts
+++ b/server/middleware/src/utils.ts
@@ -19,6 +19,6 @@ export function isOwner (account: Account, ctx: MeasureContext<SessionData>): bo
   return account.role === AccountRole.Owner || account._id === core.account.System || ctx.contextData.admin === true
 }
 
-export function isSystem (account: Account): boolean {
-  return account._id === core.account.System || account._id.startsWith('system:')
+export function isSystem (account: Account, ctx: MeasureContext<SessionData>): boolean {
+  return account._id === core.account.System || ctx.contextData.admin === true
 }

From 00783a19ddab53bbeba51b1569180f4827a9444f Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@gmail.com>
Date: Mon, 14 Oct 2024 17:34:03 +0700
Subject: [PATCH 02/12] Fix merge issue

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
 services/telegram-bot/pod-telegram-bot/src/server.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/services/telegram-bot/pod-telegram-bot/src/server.ts b/services/telegram-bot/pod-telegram-bot/src/server.ts
index 54e8a6a406..d2fb2274cc 100644
--- a/services/telegram-bot/pod-telegram-bot/src/server.ts
+++ b/services/telegram-bot/pod-telegram-bot/src/server.ts
@@ -209,7 +209,6 @@ export function createServer (
   app.post(
     '/notify',
     wrapRequest(async (req, res, token) => {
-      ctx.info('Received notification', { email: token.email })
       if (req.body == null || !Array.isArray(req.body)) {
         ctx.error('Invalid request body', { body: req.body, email: token.email })
         throw new ApiError(400)

From 3ec53392b0e2172b7ca3c745ed32e77e573b8b7c Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@users.noreply.github.com>
Date: Mon, 14 Oct 2024 22:22:19 +0700
Subject: [PATCH 03/12] UBERF-8461: Fix migration do not clean backup info
 (#6913)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
 dev/tool/src/clean.ts                | 73 ++++++++++++++--------------
 models/activity/src/migration.ts     |  6 +++
 models/attachment/src/migration.ts   | 19 +++++++-
 models/chunter/src/migration.ts      |  6 +++
 models/contact/src/migration.ts      | 11 ++++-
 models/core/src/migration.ts         |  7 +++
 models/document/src/migration.ts     |  6 +++
 models/drive/src/migration.ts        |  6 +++
 models/notification/src/migration.ts |  6 +++
 models/task/src/migration.ts         |  6 +++
 models/time/src/migration.ts         |  6 +++
 models/view/src/migration.ts         |  6 +++
 server/mongo/src/storage.ts          |  1 +
 13 files changed, 118 insertions(+), 41 deletions(-)

diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts
index bbec65bbb3..1003a81635 100644
--- a/dev/tool/src/clean.ts
+++ b/dev/tool/src/clean.ts
@@ -14,13 +14,13 @@
 //
 
 import { getAccountDB, listWorkspacesRaw } from '@hcengineering/account'
-import attachment from '@hcengineering/attachment'
 import calendar from '@hcengineering/calendar'
 import chunter, { type ChatMessage } from '@hcengineering/chunter'
 import { loadCollaborativeDoc, saveCollaborativeDoc, yDocToBuffer } from '@hcengineering/collaboration'
 import contact from '@hcengineering/contact'
 import core, {
   type ArrOf,
+  type AttachedDoc,
   type BackupClient,
   type Class,
   ClassifierKind,
@@ -45,11 +45,14 @@ import core, {
   SortingOrder,
   type Status,
   type StatusCategory,
+  type Tx,
   type TxCUD,
+  type TxCollectionCUD,
   type TxCreateDoc,
   type TxMixin,
   TxOperations,
   TxProcessor,
+  type TxRemoveDoc,
   type TxUpdateDoc,
   type WorkspaceId,
   generateId,
@@ -93,30 +96,6 @@ export async function cleanWorkspace (
 
     const hierarchy = ops.getHierarchy()
 
-    const attachments = await ops.findAll(attachment.class.Attachment, {})
-
-    const contacts = await ops.findAll(contact.class.Contact, {})
-
-    const files = new Set(
-      attachments.map((it) => it.file as string).concat(contacts.map((it) => it.avatar).filter((it) => it) as string[])
-    )
-
-    const minioList = await storageAdapter.listStream(ctx, workspaceId)
-    const toClean: string[] = []
-    while (true) {
-      const mvFiles = await minioList.next()
-      if (mvFiles.length === 0) {
-        break
-      }
-
-      for (const mv of mvFiles) {
-        if (!files.has(mv._id)) {
-          toClean.push(mv._id)
-        }
-      }
-    }
-    await storageAdapter.remove(ctx, workspaceId, toClean)
-
     if (opt.recruit) {
       const contacts = await ops.findAll(recruit.mixin.Candidate, {})
       console.log('removing Talents', contacts.length)
@@ -134,13 +113,6 @@ export async function cleanWorkspace (
         const t2 = Date.now()
         console.log('remove time:', t2 - t, filter.length)
       }
-
-      // const vacancies = await ops.findAll(recruit.class.Vacancy, {})
-      // console.log('removing vacancies', vacancies.length)
-      // for (const c of vacancies) {
-      //   console.log('Remove', c.name)
-      //   await ops.remove(c)
-      // }
     }
 
     if (opt.tracker) {
@@ -166,12 +138,39 @@ export async function cleanWorkspace (
       const db = getWorkspaceMongoDB(_client, workspaceId)
 
       if (opt.removedTx) {
-        const txes = await db.collection(DOMAIN_TX).find({}).toArray()
+        let processed = 0
+        const iterator = db.collection(DOMAIN_TX).find({})
+        while (true) {
+          const txes: Tx[] = []
 
-        for (const tx of txes) {
-          if (tx._class === core.class.TxRemoveDoc) {
-            // We need to remove all update and create operations for document
-            await db.collection(DOMAIN_TX).deleteMany({ objectId: tx.objectId })
+          const doc = await iterator.next()
+          if (doc == null) {
+            break
+          }
+          txes.push(doc as unknown as Tx)
+          if (iterator.bufferedCount() > 0) {
+            txes.push(...(iterator.readBufferedDocuments() as unknown as Tx[]))
+          }
+
+          for (const tx of txes) {
+            if (tx._class === core.class.TxRemoveDoc) {
+              // We need to remove all update and create operations for document
+              await db.collection(DOMAIN_TX).deleteMany({ objectId: (tx as TxRemoveDoc<Doc>).objectId })
+              processed++
+            }
+            if (
+              tx._class === core.class.TxCollectionCUD &&
+              (tx as TxCollectionCUD<Doc, AttachedDoc>).tx._class === core.class.TxRemoveDoc
+            ) {
+              // We need to remove all update and create operations for document
+              await db.collection(DOMAIN_TX).deleteMany({
+                'tx.objectId': ((tx as TxCollectionCUD<Doc, AttachedDoc>).tx as TxRemoveDoc<Doc>).objectId
+              })
+              processed++
+            }
+          }
+          if (processed % 1000 === 0) {
+            console.log('processed', processed)
           }
         }
       }
diff --git a/models/activity/src/migration.ts b/models/activity/src/migration.ts
index f504257d63..1f71a9b5f4 100644
--- a/models/activity/src/migration.ts
+++ b/models/activity/src/migration.ts
@@ -221,6 +221,12 @@ export const activityOperation: MigrateOperation = {
       {
         state: 'migrate-activity-markup',
         func: migrateActivityMarkup
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_ACTIVITY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },
diff --git a/models/attachment/src/migration.ts b/models/attachment/src/migration.ts
index b820fd1b74..5fead855a8 100644
--- a/models/attachment/src/migration.ts
+++ b/models/attachment/src/migration.ts
@@ -13,9 +13,24 @@
 // limitations under the License.
 //
 
-import { type MigrateOperation, type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model'
+import {
+  tryMigrate,
+  type MigrateOperation,
+  type MigrationClient,
+  type MigrationUpgradeClient
+} from '@hcengineering/model'
+import { attachmentId, DOMAIN_ATTACHMENT } from '.'
 
 export const attachmentOperation: MigrateOperation = {
-  async migrate (client: MigrationClient): Promise<void> {},
+  async migrate (client: MigrationClient): Promise<void> {
+    await tryMigrate(client, attachmentId, [
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_ATTACHMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
+      }
+    ])
+  },
   async upgrade (state: Map<string, Set<string>>, client: () => Promise<MigrationUpgradeClient>): Promise<void> {}
 }
diff --git a/models/chunter/src/migration.ts b/models/chunter/src/migration.ts
index 70d2c2151c..65438a982b 100644
--- a/models/chunter/src/migration.ts
+++ b/models/chunter/src/migration.ts
@@ -362,6 +362,12 @@ export const chunterOperation: MigrateOperation = {
             'attributeUpdates.attrKey': 'members'
           })
         }
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_CHUNTER, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },
diff --git a/models/contact/src/migration.ts b/models/contact/src/migration.ts
index 421bfb84f7..a109668aa8 100644
--- a/models/contact/src/migration.ts
+++ b/models/contact/src/migration.ts
@@ -1,5 +1,6 @@
 //
 
+import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact'
 import {
   type Class,
   type Doc,
@@ -24,9 +25,8 @@ import {
 import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity'
 import core, { DOMAIN_SPACE } from '@hcengineering/model-core'
 import { DOMAIN_VIEW } from '@hcengineering/model-view'
-import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact'
 
-import contact, { contactId, DOMAIN_CONTACT } from './index'
+import contact, { contactId, DOMAIN_CHANNEL, DOMAIN_CONTACT } from './index'
 
 async function createEmployeeEmail (client: TxOperations): Promise<void> {
   const employees = await client.findAll(contact.mixin.Employee, {})
@@ -300,6 +300,13 @@ export const contactOperation: MigrateOperation = {
       {
         state: 'create-person-spaces-v1',
         func: createPersonSpaces
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_CONTACT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+          await client.update(DOMAIN_CHANNEL, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },
diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts
index 8baa7860f5..844d1e96f0 100644
--- a/models/core/src/migration.ts
+++ b/models/core/src/migration.ts
@@ -301,6 +301,13 @@ export const coreOperation: MigrateOperation = {
       {
         state: 'collaborative-content-to-storage',
         func: migrateCollaborativeContentToStorage
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_TX, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+          await client.update(DOMAIN_SPACE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },
diff --git a/models/document/src/migration.ts b/models/document/src/migration.ts
index 9b96bfd73e..84ed089391 100644
--- a/models/document/src/migration.ts
+++ b/models/document/src/migration.ts
@@ -228,6 +228,12 @@ export const documentOperation: MigrateOperation = {
       {
         state: 'renameFields',
         func: renameFields
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_DOCUMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },
diff --git a/models/drive/src/migration.ts b/models/drive/src/migration.ts
index e2762c8b13..6ea69d90d2 100644
--- a/models/drive/src/migration.ts
+++ b/models/drive/src/migration.ts
@@ -132,6 +132,12 @@ export const driveOperation: MigrateOperation = {
       {
         state: 'renameFields',
         func: renameFields
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_DRIVE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },
diff --git a/models/notification/src/migration.ts b/models/notification/src/migration.ts
index 7693cfb25d..4c67119729 100644
--- a/models/notification/src/migration.ts
+++ b/models/notification/src/migration.ts
@@ -393,6 +393,12 @@ export const notificationOperation: MigrateOperation = {
             { hidden: false }
           )
         }
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_DOC_NOTIFY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
 
diff --git a/models/task/src/migration.ts b/models/task/src/migration.ts
index 93f7b7c0eb..a9f45ccacc 100644
--- a/models/task/src/migration.ts
+++ b/models/task/src/migration.ts
@@ -584,6 +584,12 @@ export const taskOperation: MigrateOperation = {
         func: async (client: MigrationClient) => {
           await migrateSpace(client, task.space.Sequence, core.space.Workspace, [DOMAIN_KANBAN])
         }
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_TASK, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },
diff --git a/models/time/src/migration.ts b/models/time/src/migration.ts
index 91e282515c..481e28dfe0 100644
--- a/models/time/src/migration.ts
+++ b/models/time/src/migration.ts
@@ -172,6 +172,12 @@ export const timeOperation: MigrateOperation = {
         func: async (client) => {
           await fillProps(client)
         }
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_TIME, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },
diff --git a/models/view/src/migration.ts b/models/view/src/migration.ts
index f5587c0547..572a60d8d7 100644
--- a/models/view/src/migration.ts
+++ b/models/view/src/migration.ts
@@ -86,6 +86,12 @@ export const viewOperation: MigrateOperation = {
       {
         state: 'remove-done-state-filter',
         func: removeDoneStateFilter
+      },
+      {
+        state: 'fix-rename-backups',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(DOMAIN_VIEW, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+        }
       }
     ])
   },
diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts
index 1eacbf7e11..7e4f1cd848 100644
--- a/server/mongo/src/storage.ts
+++ b/server/mongo/src/storage.ts
@@ -251,6 +251,7 @@ abstract class MongoAdapterBase implements DbAdapter {
     operations: DocumentUpdate<T>
   ): Promise<void> {
     if (isOperator(operations)) {
+      await this.db.collection(domain).updateMany(this.translateRawQuery(query), { $set: { '%hash%': null } })
       await this.db
         .collection(domain)
         .updateMany(this.translateRawQuery(query), { ...operations } as unknown as UpdateFilter<Document>)

From 568dd2f47eec2ab98d72d4283afe429a25caee61 Mon Sep 17 00:00:00 2001
From: Denis Bykhov <bykhov.denis@gmail.com>
Date: Tue, 15 Oct 2024 00:44:06 +0500
Subject: [PATCH 04/12] Fix calendar and gmail services (#6918)

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>
---
 .../calendar/pod-calendar/src/calendar.ts     |  1 -
 .../pod-calendar/src/calendarController.ts    | 55 +++++++++++++++---
 services/calendar/pod-calendar/src/config.ts  |  5 +-
 services/gmail/pod-gmail/src/config.ts        |  5 +-
 services/gmail/pod-gmail/src/gmail.ts         |  2 +-
 .../gmail/pod-gmail/src/gmailController.ts    | 57 ++++++++++++++++---
 .../gmail/pod-gmail/src/workspaceClient.ts    |  7 ++-
 7 files changed, 112 insertions(+), 20 deletions(-)

diff --git a/services/calendar/pod-calendar/src/calendar.ts b/services/calendar/pod-calendar/src/calendar.ts
index 05b0672ae3..3e1b75b90a 100644
--- a/services/calendar/pod-calendar/src/calendar.ts
+++ b/services/calendar/pod-calendar/src/calendar.ts
@@ -102,7 +102,6 @@ export class CalendarClient {
       await calendarClient.setToken(user)
       await calendarClient.refreshToken()
       await calendarClient.addClient()
-      void calendarClient.startSync()
     }
     return calendarClient
   }
diff --git a/services/calendar/pod-calendar/src/calendarController.ts b/services/calendar/pod-calendar/src/calendarController.ts
index b13d08fe23..55b5f2e3e6 100644
--- a/services/calendar/pod-calendar/src/calendarController.ts
+++ b/services/calendar/pod-calendar/src/calendarController.ts
@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 
-import { Account, Ref } from '@hcengineering/core'
+import { Account, RateLimiter, Ref } from '@hcengineering/core'
 import { type Db } from 'mongodb'
 import { type CalendarClient } from './calendar'
 import config from './config'
@@ -25,6 +25,7 @@ export class CalendarController {
 
   private readonly credentials: ProjectCredentials
   private readonly clients: Map<string, CalendarClient[]> = new Map<string, CalendarClient[]>()
+  private readonly initLimitter = new RateLimiter(config.InitLimit)
 
   protected static _instance: CalendarController
 
@@ -43,17 +44,57 @@ export class CalendarController {
 
   async startAll (): Promise<void> {
     const tokens = await this.mongo.collection<Token>('tokens').find().toArray()
+    const groups = new Map<string, Token[]>()
+    console.log('start calendar service', tokens.length)
     for (const token of tokens) {
-      try {
-        await this.createClient(token)
-      } catch (err) {
-        console.error(`Couldn't create client for ${token.workspace} ${token.userId}`)
+      const group = groups.get(token.workspace)
+      if (group === undefined) {
+        groups.set(token.workspace, [token])
+      } else {
+        group.push(token)
+        groups.set(token.workspace, group)
       }
     }
 
-    for (const client of this.workspaces.values()) {
-      void client.sync()
+    const limiter = new RateLimiter(config.InitLimit)
+
+    for (const [workspace, tokens] of groups) {
+      await limiter.add(async () => {
+        const startPromise = this.startWorkspace(workspace, tokens)
+        const timeoutPromise = new Promise<void>((resolve) => {
+          setTimeout(() => {
+            resolve()
+          }, 60000)
+        })
+        await Promise.race([startPromise, timeoutPromise])
+      })
     }
+
+    await limiter.waitProcessing()
+    console.log('Calendar service started')
+  }
+
+  async startWorkspace (workspace: string, tokens: Token[]): Promise<void> {
+    const workspaceClient = await this.getWorkspaceClient(workspace)
+    const clients: CalendarClient[] = []
+    for (const token of tokens) {
+      try {
+        const timeout = setTimeout(() => {
+          console.log('init client hang', token.workspace, token.userId)
+        }, 60000)
+        const client = await workspaceClient.createCalendarClient(token)
+        clearTimeout(timeout)
+        clients.push(client)
+      } catch (err) {
+        console.error(`Couldn't create client for ${workspace} ${token.userId}`)
+      }
+    }
+    for (const client of clients) {
+      void this.initLimitter.add(async () => {
+        await client.startSync()
+      })
+    }
+    void workspaceClient.sync()
   }
 
   push (email: string, mode: 'events' | 'calendar', calendarId?: string): void {
diff --git a/services/calendar/pod-calendar/src/config.ts b/services/calendar/pod-calendar/src/config.ts
index 1647c9cfd9..e93fb08446 100644
--- a/services/calendar/pod-calendar/src/config.ts
+++ b/services/calendar/pod-calendar/src/config.ts
@@ -24,6 +24,7 @@ interface Config {
   Credentials: string
   WATCH_URL: string
   SystemEmail: string
+  InitLimit: number
 }
 
 const envMap: { [key in keyof Config]: string } = {
@@ -37,7 +38,8 @@ const envMap: { [key in keyof Config]: string } = {
   Secret: 'SECRET',
   Credentials: 'Credentials',
   SystemEmail: 'SYSTEM_EMAIL',
-  WATCH_URL: 'WATCH_URL'
+  WATCH_URL: 'WATCH_URL',
+  InitLimit: 'INIT_LIMIT'
 }
 
 const parseNumber = (str: string | undefined): number | undefined => (str !== undefined ? Number(str) : undefined)
@@ -52,6 +54,7 @@ const config: Config = (() => {
     Secret: process.env[envMap.Secret],
     SystemEmail: process.env[envMap.SystemEmail] ?? 'anticrm@hc.engineering',
     Credentials: process.env[envMap.Credentials],
+    InitLimit: parseNumber(process.env[envMap.InitLimit]) ?? 50,
     WATCH_URL: process.env[envMap.WATCH_URL]
   }
 
diff --git a/services/gmail/pod-gmail/src/config.ts b/services/gmail/pod-gmail/src/config.ts
index faba0c357b..3d98317309 100644
--- a/services/gmail/pod-gmail/src/config.ts
+++ b/services/gmail/pod-gmail/src/config.ts
@@ -26,6 +26,7 @@ interface Config {
   WATCH_TOPIC_NAME: string
   SystemEmail: string
   FooterMessage: string
+  InitLimit: number
 }
 
 const envMap: { [key in keyof Config]: string } = {
@@ -40,7 +41,8 @@ const envMap: { [key in keyof Config]: string } = {
   Credentials: 'Credentials',
   SystemEmail: 'SYSTEM_EMAIL',
   WATCH_TOPIC_NAME: 'WATCH_TOPIC_NAME',
-  FooterMessage: 'FOOTER_MESSAGE'
+  FooterMessage: 'FOOTER_MESSAGE',
+  InitLimit: 'INIT_LIMIT'
 }
 
 const parseNumber = (str: string | undefined): number | undefined => (str !== undefined ? Number(str) : undefined)
@@ -56,6 +58,7 @@ const config: Config = (() => {
     SystemEmail: process.env[envMap.SystemEmail] ?? 'anticrm@hc.engineering',
     Credentials: process.env[envMap.Credentials],
     WATCH_TOPIC_NAME: process.env[envMap.WATCH_TOPIC_NAME],
+    InitLimit: parseNumber(process.env[envMap.InitLimit]) ?? 50,
     FooterMessage: process.env[envMap.FooterMessage] ?? '<br><br><p>Sent via <a href="https://huly.io">Huly</a></p>'
   }
 
diff --git a/services/gmail/pod-gmail/src/gmail.ts b/services/gmail/pod-gmail/src/gmail.ts
index f8d84a1789..8d7c80be4e 100644
--- a/services/gmail/pod-gmail/src/gmail.ts
+++ b/services/gmail/pod-gmail/src/gmail.ts
@@ -211,6 +211,7 @@ export class GmailClient {
     await this.setToken(token.tokens)
     await this.refreshToken()
     await this.addClient()
+    void this.startSync()
     void this.getNewMessagesAfterAuth()
 
     const me = await this.getMe()
@@ -385,7 +386,6 @@ export class GmailClient {
   private async addClient (): Promise<void> {
     try {
       const me = await this.getMe()
-      void this.startSync()
       const controller = GmailController.getGmailController()
       controller.addClient(me, this)
     } catch (err) {
diff --git a/services/gmail/pod-gmail/src/gmailController.ts b/services/gmail/pod-gmail/src/gmailController.ts
index 62a29286e4..8be43cdd5c 100644
--- a/services/gmail/pod-gmail/src/gmailController.ts
+++ b/services/gmail/pod-gmail/src/gmailController.ts
@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 
-import { MeasureContext } from '@hcengineering/core'
+import { MeasureContext, RateLimiter } from '@hcengineering/core'
 import type { StorageAdapter } from '@hcengineering/server-core'
 
 import { type Db } from 'mongodb'
@@ -28,6 +28,7 @@ export class GmailController {
 
   private readonly credentials: ProjectCredentials
   private readonly clients: Map<string, GmailClient[]> = new Map<string, GmailClient[]>()
+  private readonly initLimitter = new RateLimiter(config.InitLimit)
 
   protected static _instance: GmailController
 
@@ -56,19 +57,57 @@ export class GmailController {
 
   async startAll (): Promise<void> {
     const tokens = await this.mongo.collection<Token>('tokens').find().toArray()
+    const groups = new Map<string, Token[]>()
+    console.log('start gmail service', tokens.length)
     for (const token of tokens) {
-      try {
-        await this.createClient(token)
-      } catch (err) {
-        console.error(`Couldn't create client for ${token.workspace} ${token.userId}`)
+      const group = groups.get(token.workspace)
+      if (group === undefined) {
+        groups.set(token.workspace, [token])
+      } else {
+        group.push(token)
+        groups.set(token.workspace, group)
       }
     }
 
-    for (const client of this.workspaces.values()) {
-      void client.checkUsers().then(async () => {
-        await client.getNewMessages()
+    const limiter = new RateLimiter(config.InitLimit)
+    for (const [workspace, tokens] of groups) {
+      await limiter.add(async () => {
+        const startPromise = this.startWorkspace(workspace, tokens)
+        const timeoutPromise = new Promise<void>((resolve) => {
+          setTimeout(() => {
+            resolve()
+          }, 60000)
+        })
+        await Promise.race([startPromise, timeoutPromise])
       })
     }
+
+    await limiter.waitProcessing()
+  }
+
+  async startWorkspace (workspace: string, tokens: Token[]): Promise<void> {
+    const workspaceClient = await this.getWorkspaceClient(workspace)
+    const clients: GmailClient[] = []
+    for (const token of tokens) {
+      try {
+        const timeout = setTimeout(() => {
+          console.log('init client hang', token.workspace, token.userId)
+        }, 60000)
+        const client = await workspaceClient.createGmailClient(token)
+        clearTimeout(timeout)
+        clients.push(client)
+      } catch (err) {
+        console.error(`Couldn't create client for ${workspace} ${token.userId}`)
+      }
+    }
+    for (const client of clients) {
+      void this.initLimitter.add(async () => {
+        await client.startSync()
+      })
+    }
+    void workspaceClient.checkUsers().then(async () => {
+      await workspaceClient.getNewMessages()
+    })
   }
 
   push (message: string): void {
@@ -121,8 +160,10 @@ export class GmailController {
     let res = this.workspaces.get(workspace)
     if (res === undefined) {
       try {
+        console.log('create workspace worker for', workspace)
         res = await WorkspaceClient.create(this.ctx, this.credentials, this.mongo, this.storageAdapter, workspace)
         this.workspaces.set(workspace, res)
+        console.log('created workspace worker for', workspace)
       } catch (err) {
         console.error(`Couldn't create workspace worker for ${workspace}, reason: `, err)
         throw err
diff --git a/services/gmail/pod-gmail/src/workspaceClient.ts b/services/gmail/pod-gmail/src/workspaceClient.ts
index d18c7c78c5..c5df09a49a 100644
--- a/services/gmail/pod-gmail/src/workspaceClient.ts
+++ b/services/gmail/pod-gmail/src/workspaceClient.ts
@@ -159,11 +159,15 @@ export class WorkspaceClient {
     const newMessages = await this.client.findAll(gmailP.class.NewMessage, {
       status: 'new'
     })
+    console.log('get new messages, recieved', this.workspace, newMessages.length)
     await this.subscribeMessages()
     for (const message of newMessages) {
-      const client = this.getGmailClient(message.from ?? message.createdBy ?? message.modifiedBy)
+      const from = message.from ?? message.createdBy ?? message.modifiedBy
+      const client = this.getGmailClient(from)
       if (client !== undefined) {
         await client.createMessage(message)
+      } else {
+        console.log('client not found, skip message', this.workspace, from, message._id)
       }
     }
   }
@@ -342,6 +346,7 @@ export class WorkspaceClient {
         await this.txEmployeeHandler(tx)
       }
     })
+    console.log('deactivate users', this.workspace, accounts.length)
   }
 
   private async deactivateUser (acc: PersonAccount): Promise<void> {

From b5b4d23629d42d34ccf3dbea079d8a27e33b1224 Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@users.noreply.github.com>
Date: Tue, 15 Oct 2024 12:04:39 +0700
Subject: [PATCH 05/12] UBERF-8469: Fix exit from github service (#6921)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
 services/github/pod-github/package.json       |  4 ++--
 services/github/pod-github/run.sh             | 13 +++++++++++++
 services/github/pod-github/src/index.ts       | 15 +++++++++++----
 services/github/pod-github/src/platform.ts    |  1 +
 services/github/pod-github/src/server.ts      |  9 +++++++--
 .../github/pod-github/src/sync/comments.ts    |  6 ++++++
 .../github/pod-github/src/sync/issueBase.ts   |  2 +-
 services/github/pod-github/src/sync/issues.ts |  9 +++++++++
 .../github/pod-github/src/sync/projects.ts    |  6 ++++++
 .../pod-github/src/sync/pullrequests.ts       |  6 ++++++
 services/github/pod-github/src/sync/users.ts  |  6 ++++++
 services/github/pod-github/src/types.ts       |  2 ++
 services/github/pod-github/src/worker.ts      | 19 +++++++++++++++++++
 13 files changed, 89 insertions(+), 9 deletions(-)
 create mode 100755 services/github/pod-github/run.sh

diff --git a/services/github/pod-github/package.json b/services/github/pod-github/package.json
index 39de621832..3dc820dff0 100644
--- a/services/github/pod-github/package.json
+++ b/services/github/pod-github/package.json
@@ -17,12 +17,12 @@
     "_phase:bundle": "rushx bundle",
     "_phase:docker-build": "rushx docker:build",
     "_phase:docker-staging": "rushx docker:staging",
-    "bundle": "mkdir -p bundle && esbuild src/index.ts --bundle --platform=node --outfile=bundle/bundle.js --log-level=error --sourcemap=external",
+    "bundle": "mkdir -p bundle && esbuild src/index.ts --keep-names --bundle --platform=node --outfile=bundle/bundle.js --log-level=error --sourcemap=external",
     "docker:build": "../../../common/scripts/docker_build.sh hardcoreeng/github",
     "docker:staging": "../../../common/scripts/docker_tag.sh hardcoreeng/github staging",
     "docker:push": "../../../common/scripts/docker_tag.sh hardcoreeng/github",
     "docker:tbuild": "rush bundle --to @hcengineering/pod-github && docker build -t hardcoreeng/github . --platform=linux/amd64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/github",
-    "run-local": "cross-env APP_ID=$(cat ../../../../uberflow_private/appid) PRIVATE_KEY=\"$(cat ../../../../uberflow_private/private-key.pem)\" CLIENT_ID=$(cat ../../../../uberflow_private/client-id) CLIENT_SECRET=$(cat ../../../../uberflow_private/client-secret) SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000/ COLLABORATOR_URL=http://localhost:3078 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ts-node src/index.ts",
+    "run-local": "ruh.sh",
     "format": "format src",
     "_phase:build": "compile transpile src",
     "_phase:test": "jest --passWithNoTests --silent",
diff --git a/services/github/pod-github/run.sh b/services/github/pod-github/run.sh
new file mode 100755
index 0000000000..ebc9faf4f9
--- /dev/null
+++ b/services/github/pod-github/run.sh
@@ -0,0 +1,13 @@
+export APP_ID="$POD_GITHUB_APPID"
+export CLIENT_ID="$POD_GITHUB_CLIENTID"
+export CLIENT_SECRET="$POD_GITHUB_CLIENT_SECRET"
+export PRIVATE_KEY="$POD_GITHUB_PRIVATE_KEY"
+export SERVER_SECRET=secret
+export ACCOUNTS_URL=http://localhost:3000
+export COLLABORATOR_URL=http://localhost:3078
+export MINIO_ACCESS_KEY=minioadminchmo
+export MINIO_SECRET_KEY=minioadmin
+export MINIO_ENDPOINT=localhost
+export MONGO_URL=mongodb://localhost:27017
+rush bundle --to @hcengineering/pod-github
+node $@ bundle/bundle.js
\ No newline at end of file
diff --git a/services/github/pod-github/src/index.ts b/services/github/pod-github/src/index.ts
index fa038ef824..e154398a93 100644
--- a/services/github/pod-github/src/index.ts
+++ b/services/github/pod-github/src/index.ts
@@ -2,14 +2,14 @@
 // Copyright © 2023 Hardcore Engineering Inc.
 //
 
-import { MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core'
+import { Analytics } from '@hcengineering/analytics'
 import { SplitLogger, configureAnalytics } from '@hcengineering/analytics-service'
+import { MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core'
+import { loadBrandingMap } from '@hcengineering/server-core'
 import { writeFile } from 'fs/promises'
 import { join } from 'path'
 import config from './config'
 import { start } from './server'
-import { Analytics } from '@hcengineering/analytics'
-import { loadBrandingMap } from '@hcengineering/server-core'
 
 // Load and inc startID, to have easy logs.
 
@@ -39,11 +39,18 @@ const intTimer = setInterval(() => {
   }
 }, 30000)
 
-void start(metricsContext, loadBrandingMap(config.BrandingPath))
+let doOnClose: () => Promise<void> = async () => {}
+
+void start(metricsContext, loadBrandingMap(config.BrandingPath)).then((r) => {
+  doOnClose = r
+})
 
 const onClose = (): void => {
   clearInterval(intTimer)
   metricsContext.info('Closed')
+  void doOnClose().then((r) => {
+    process.exit(0)
+  })
 }
 
 process.on('uncaughtException', (e) => {
diff --git a/services/github/pod-github/src/platform.ts b/services/github/pod-github/src/platform.ts
index f525fdf803..6b1c6ed307 100644
--- a/services/github/pod-github/src/platform.ts
+++ b/services/github/pod-github/src/platform.ts
@@ -96,6 +96,7 @@ export class PlatformWorker {
 
   async close (): Promise<void> {
     this.canceled = true
+    clearInterval(this.periodicSyncInterval)
     await Promise.all(
       [...this.clients.values()].map(async (worker) => {
         await worker.close()
diff --git a/services/github/pod-github/src/server.ts b/services/github/pod-github/src/server.ts
index b0d8d0d9ae..df41cd0a25 100644
--- a/services/github/pod-github/src/server.ts
+++ b/services/github/pod-github/src/server.ts
@@ -18,7 +18,7 @@ import { decodeToken } from '@hcengineering/server-token'
 /**
  * @public
  */
-export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Promise<void> {
+export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Promise<() => Promise<void>> {
   // Create an authenticated Octokit client authenticated as a GitHub App
   ctx.info('Running Huly Github integration', { appId: config.AppID, clientID: config.ClientID })
 
@@ -184,8 +184,13 @@ export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Pro
     }
   })
 
-  app.listen(port, () => {
+  const server = app.listen(port, () => {
     ctx.info(`Server is listening for events at: ${localWebhookUrl}`)
     ctx.info('Press Ctrl + C to quit.')
   })
+
+  return async () => {
+    await worker.close()
+    server.close()
+  }
 }
diff --git a/services/github/pod-github/src/sync/comments.ts b/services/github/pod-github/src/sync/comments.ts
index cf525d0903..3492c30599 100644
--- a/services/github/pod-github/src/sync/comments.ts
+++ b/services/github/pod-github/src/sync/comments.ts
@@ -457,6 +457,9 @@ export class CommentSyncManager implements DocSyncManager {
     repositories: GithubIntegrationRepository[]
   ): Promise<void> {
     for (const repo of repositories) {
+      if (this.provider.isClosing()) {
+        break
+      }
       const syncKey = `${repo._id}:comment`
       if (repo.githubProject === undefined || !repo.enabled || integration.synchronized.has(syncKey)) {
         if (!repo.enabled) {
@@ -487,6 +490,9 @@ export class CommentSyncManager implements DocSyncManager {
       })
       try {
         for await (const data of i) {
+          if (this.provider.isClosing()) {
+            break
+          }
           const comments: CommentExternalData[] = data.data as any
           this.ctx.info('retrieve comments for', {
             repo: repo.name,
diff --git a/services/github/pod-github/src/sync/issueBase.ts b/services/github/pod-github/src/sync/issueBase.ts
index f2e9c9278a..0ff45f0f96 100644
--- a/services/github/pod-github/src/sync/issueBase.ts
+++ b/services/github/pod-github/src/sync/issueBase.ts
@@ -815,7 +815,7 @@ export abstract class IssueSyncManagerBase {
 
       // Collect field update.
       for (const [k, v] of Object.entries(platformUpdate)) {
-        const mapping = target.mappings.find((it) => it.name === k)
+        const mapping = target.mappings.filter((it) => it != null).find((it) => it.name === k)
         if (mapping === undefined) {
           continue
         }
diff --git a/services/github/pod-github/src/sync/issues.ts b/services/github/pod-github/src/sync/issues.ts
index c981910720..290ce7ed15 100644
--- a/services/github/pod-github/src/sync/issues.ts
+++ b/services/github/pod-github/src/sync/issues.ts
@@ -985,6 +985,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
     let partsize = 50
     try {
       while (true) {
+        if (this.provider.isClosing()) {
+          break
+        }
         const idsPart = ids.splice(0, partsize)
         if (idsPart.length === 0) {
           break
@@ -1080,6 +1083,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
     repositories: GithubIntegrationRepository[]
   ): Promise<void> {
     for (const repo of repositories) {
+      if (this.provider.isClosing()) {
+        break
+      }
       const prj = projects.find((it) => repo.githubProject === it._id)
       if (prj === undefined) {
         continue
@@ -1128,6 +1134,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
       )
       try {
         for await (const data of i) {
+          if (this.provider.isClosing()) {
+            break
+          }
           const issues: IssueExternalData[] = data.repository.issues.nodes
           if (issues.some((issue) => issue.url === undefined && Object.keys(issue).length === 0)) {
             this.ctx.error('empty document content', {
diff --git a/services/github/pod-github/src/sync/projects.ts b/services/github/pod-github/src/sync/projects.ts
index 6833820794..4748b1e41a 100644
--- a/services/github/pod-github/src/sync/projects.ts
+++ b/services/github/pod-github/src/sync/projects.ts
@@ -376,6 +376,9 @@ export class ProjectsSyncManager implements DocSyncManager {
     repositories: GithubIntegrationRepository[]
   ): Promise<void> {
     for (const prj of projects) {
+      if (this.provider.isClosing()) {
+        break
+      }
       // Wait global project sync
       await integration.syncLock.get(prj._id)
 
@@ -449,6 +452,9 @@ export class ProjectsSyncManager implements DocSyncManager {
           (it) => it.space === prj._id
         )
         for (const m of milestones) {
+          if (this.provider.isClosing()) {
+            break
+          }
           try {
             let { projectStructure, wasUpdates } = await this.ctx.withLog(
               'update project structure',
diff --git a/services/github/pod-github/src/sync/pullrequests.ts b/services/github/pod-github/src/sync/pullrequests.ts
index 8ef9fd275d..51d3e78d54 100644
--- a/services/github/pod-github/src/sync/pullrequests.ts
+++ b/services/github/pod-github/src/sync/pullrequests.ts
@@ -1433,6 +1433,9 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
     repositories: GithubIntegrationRepository[]
   ): Promise<void> {
     for (const repo of repositories) {
+      if (this.provider.isClosing()) {
+        break
+      }
       const prj = projects.find((it) => repo.githubProject === it._id)
       if (prj === undefined) {
         continue
@@ -1518,6 +1521,9 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
         }
       )
       for await (const data of pullRequestIterator) {
+        if (this.provider.isClosing()) {
+          break
+        }
         const issues: PullRequestExternalData[] = data.repository.pullRequests.nodes
         this.ctx.info('retrieve pull requests for', {
           repo: repo.name,
diff --git a/services/github/pod-github/src/sync/users.ts b/services/github/pod-github/src/sync/users.ts
index 94f5d43b72..b018dce4fa 100644
--- a/services/github/pod-github/src/sync/users.ts
+++ b/services/github/pod-github/src/sync/users.ts
@@ -60,6 +60,9 @@ export class UsersSyncManager implements DocSyncManager {
     repositories: GithubIntegrationRepository[]
   ): Promise<void> {
     for (const repo of repositories) {
+      if (this.provider.isClosing()) {
+        break
+      }
       const syncKey = `${repo._id}:users`
       if (
         repo.githubProject === undefined ||
@@ -107,6 +110,9 @@ export class UsersSyncManager implements DocSyncManager {
     )
     try {
       for await (const data of assignableUsersIterator) {
+        if (this.provider.isClosing()) {
+          break
+        }
         const users: UserInfo[] = data.repository[key]?.nodes ?? []
         for (const d of users) {
           if (d.login !== undefined) {
diff --git a/services/github/pod-github/src/types.ts b/services/github/pod-github/src/types.ts
index 13d33177b5..f4e24fddc9 100644
--- a/services/github/pod-github/src/types.ts
+++ b/services/github/pod-github/src/types.ts
@@ -129,6 +129,8 @@ export interface IntegrationManager {
   getProjectRepositories: (space: Ref<Space>) => Promise<GithubIntegrationRepository[]>
 
   getRepositoryById: (ref?: Ref<GithubIntegrationRepository> | null) => Promise<GithubIntegrationRepository | undefined>
+
+  isClosing: () => boolean
 }
 
 export type ExternalSyncField = 'externalVersion' | 'derivedVersion'
diff --git a/services/github/pod-github/src/worker.ts b/services/github/pod-github/src/worker.ts
index db5f974876..1ef7acdc63 100644
--- a/services/github/pod-github/src/worker.ts
+++ b/services/github/pod-github/src/worker.ts
@@ -117,11 +117,18 @@ export class GithubWorker implements IntegrationManager {
 
   personMapper: UsersSyncManager
 
+  isClosing (): boolean {
+    return this.closing
+  }
+
   async close (): Promise<void> {
     clearInterval(this.periodicTimer)
 
     this.closing = true
+    this.ctx.warn('Closing', { workspace: this.workspace.name })
+    this.triggerSync()
     await this.syncPromise
+    this.ctx.warn('ClosingDone', { workspace: this.workspace.name })
     await this.client.close()
   }
 
@@ -1347,6 +1354,9 @@ export class GithubWorker implements IntegrationManager {
   }
 
   private async waitChanges (): Promise<void> {
+    if (this.closing) {
+      return
+    }
     if (this.triggerRequests > 0 || this.updateRequests > 0) {
       this.ctx.info('Trigger check pending:', {
         requests: this.triggerRequests,
@@ -1404,6 +1414,9 @@ export class GithubWorker implements IntegrationManager {
   async _performFullSync (): Promise<void> {
     // Wait previous active sync
     for (const integration of this.integrations.values()) {
+      if (this.closing) {
+        break
+      }
       await this.ctx.withLog(
         'external sync',
         { installation: integration.installationName, workspace: this.workspace.name },
@@ -1442,6 +1455,9 @@ export class GithubWorker implements IntegrationManager {
           // Cleanup broken synchronized documents
 
           while (true) {
+            if (this.closing) {
+              break
+            }
             const withError = await derivedClient.findAll<any>(
               github.class.DocSyncInfo,
               { error: { $ne: null }, url: null },
@@ -1458,6 +1474,9 @@ export class GithubWorker implements IntegrationManager {
           }
 
           for (const { _class, mapper } of this.mappers) {
+            if (this.closing) {
+              break
+            }
             await this.ctx.withLog(
               'external sync',
               { _class: _class.join(', '), workspace: this.workspace.name },

From b32e44135176502f16874c5261daf4266ea72f83 Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@users.noreply.github.com>
Date: Tue, 15 Oct 2024 14:50:53 +0700
Subject: [PATCH 06/12] Fix under maintenance warning message (#6930)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
 plugins/workbench-assets/lang/en.json | 2 +-
 plugins/workbench-assets/lang/es.json | 2 +-
 plugins/workbench-assets/lang/fr.json | 2 +-
 plugins/workbench-assets/lang/pt.json | 2 +-
 plugins/workbench-assets/lang/ru.json | 2 +-
 plugins/workbench-assets/lang/zh.json | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/plugins/workbench-assets/lang/en.json b/plugins/workbench-assets/lang/en.json
index 4064971be0..101dec2088 100644
--- a/plugins/workbench-assets/lang/en.json
+++ b/plugins/workbench-assets/lang/en.json
@@ -24,7 +24,7 @@
     "OpenInNewTab": "Open in a new tab",
     "NewVersionAvailable": "New version is available",
     "PleaseUpdate": "Please update",
-    "ServerUnderMaintenance": "Server is under maintenance",
+    "ServerUnderMaintenance": "Preparing workspace for new version...",
     "MobileNotSupported": "Sorry, mobile devices support coming soon. In the meantime, please use Desktop",
     "LogInAnyway": "Log in anyway",
     "WorkspaceCreating": "Creation in progress...",
diff --git a/plugins/workbench-assets/lang/es.json b/plugins/workbench-assets/lang/es.json
index 0bd6e7f4ef..112218c6a2 100644
--- a/plugins/workbench-assets/lang/es.json
+++ b/plugins/workbench-assets/lang/es.json
@@ -24,7 +24,7 @@
     "OpenInNewTab": "Abrir en una nueva pestaña",
     "NewVersionAvailable": "Nueva versión disponible",
     "PleaseUpdate": "Por favor, actualice",
-    "ServerUnderMaintenance": "El servidor está en mantenimiento",
+    "ServerUnderMaintenance": "Preparando el espacio de trabajo para la nueva versión...",
     "MobileNotSupported": "Disculpa, el soporte para dispositivos móviles estará disponible próximamente. Mientras tanto, por favor usa el escritorio.",
     "LogInAnyway": "Iniciar sesión de todas formas",
     "WorkspaceCreating": "Creation in progress...",
diff --git a/plugins/workbench-assets/lang/fr.json b/plugins/workbench-assets/lang/fr.json
index f2b727f07a..5c363c5f14 100644
--- a/plugins/workbench-assets/lang/fr.json
+++ b/plugins/workbench-assets/lang/fr.json
@@ -24,7 +24,7 @@
     "OpenInNewTab": "Ouvrir dans un nouvel onglet",
     "NewVersionAvailable": "Nouvelle version disponible",
     "PleaseUpdate": "Veuillez mettre à jour",
-    "ServerUnderMaintenance": "Le serveur est en maintenance",
+    "ServerUnderMaintenance": "Préparation de l'espace de travail pour la nouvelle version...",
     "MobileNotSupported": "Désolé, le support pour les appareils mobiles arrive bientôt. En attendant, veuillez utiliser un ordinateur de bureau",
     "LogInAnyway": "Se connecter quand même",
     "WorkspaceCreating": "Création en cours...",
diff --git a/plugins/workbench-assets/lang/pt.json b/plugins/workbench-assets/lang/pt.json
index a7df3d0e14..853411388e 100644
--- a/plugins/workbench-assets/lang/pt.json
+++ b/plugins/workbench-assets/lang/pt.json
@@ -24,7 +24,7 @@
     "OpenInNewTab": "Abrir num novo separador",
     "NewVersionAvailable": "Nova versão disponível",
     "PleaseUpdate": "Atualize",
-    "ServerUnderMaintenance": "Servidor em manutenção",
+    "ServerUnderMaintenance": "Preparando o espaço de trabalho para a nova versão...",
     "MobileNotSupported": "Desculpe, o suporte para dispositivos móveis estará disponível em breve. Enquanto isso, por favor, use o Desktop.",
     "LogInAnyway": "Entrar de qualquer maneira",
     "WorkspaceCreating": "Creation in progress...",
diff --git a/plugins/workbench-assets/lang/ru.json b/plugins/workbench-assets/lang/ru.json
index f2da9f6c74..1eb03d0a1f 100644
--- a/plugins/workbench-assets/lang/ru.json
+++ b/plugins/workbench-assets/lang/ru.json
@@ -24,7 +24,7 @@
     "OpenInNewTab": "Открыть в новой вкладке",
     "NewVersionAvailable": "Доступна новая версия",
     "PleaseUpdate": "Пожалуйста, обновите приложение",
-    "ServerUnderMaintenance": "Обслуживание сервера",
+    "ServerUnderMaintenance": "Подготовка рабочего пространства для новой версии...",
     "MobileNotSupported": "Простите, поддержка мобильных устройств скоро будет доступна. Пока воспользуйтесь компьютером.",
     "LogInAnyway": "Все равно войти",
     "WorkspaceCreating": "Пространство создается...",
diff --git a/plugins/workbench-assets/lang/zh.json b/plugins/workbench-assets/lang/zh.json
index 416be8dd78..12b9514576 100644
--- a/plugins/workbench-assets/lang/zh.json
+++ b/plugins/workbench-assets/lang/zh.json
@@ -24,7 +24,7 @@
     "OpenInNewTab": "在新标签页中打开",
     "NewVersionAvailable": "有新版本可用",
     "PleaseUpdate": "请更新",
-    "ServerUnderMaintenance": "服务器正在维护中",
+    "ServerUnderMaintenance": "正在为新版本准备工作区...",
     "MobileNotSupported": "抱歉，移动设备支持即将推出。在此期间，请使用桌面设备",
     "LogInAnyway": "仍然登录",
     "WorkspaceCreating": "创建进行中...",

From a10f8cf1a27a4ecbe0636ed6b4fcfb19f4ad5e1f Mon Sep 17 00:00:00 2001
From: Alexander Onnikov <aonnikov@hardcoreeng.com>
Date: Tue, 15 Oct 2024 16:55:11 +0700
Subject: [PATCH 07/12] fix: use const slack link (#6935)

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
---
 plugins/support/src/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/support/src/index.ts b/plugins/support/src/index.ts
index 712abc96d4..47317ebdc8 100644
--- a/plugins/support/src/index.ts
+++ b/plugins/support/src/index.ts
@@ -21,7 +21,7 @@ import { SupportClientFactory, SupportConversation, SupportSystem } from './type
 export * from './types'
 export { deleteSupportConversation, updateSupportConversation } from './utils'
 
-export const supportLink = 'https://join.slack.com/t/hulycommunity/shared_invite/zt-2cyrevz8g-AGqEDZNsujbn4wHOWd7myg'
+export const supportLink = 'https://huly.link/slack'
 export const reportBugLink = 'https://github.com/hcengineering/platform/issues/new'
 export const docsLink = 'http://docs.huly.io/'
 export const privacyPolicyLink = 'https://v1.huly.io/legal/privacy/'

From 9eb914f5985629da5efa88ed6529a99ef82c6a89 Mon Sep 17 00:00:00 2001
From: Kristina <kristin.fefelova@gmail.com>
Date: Tue, 15 Oct 2024 23:38:52 +0400
Subject: [PATCH 08/12] Fix chat navigator font size (#6947)

Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com>
---
 packages/ui/src/components/NavItem.svelte                      | 3 +--
 .../src/components/chat/navigator/ChatNavItem.svelte           | 1 -
 .../src/components/chat/navigator/ChatNavSection.svelte        | 3 +--
 .../src/components/chat/navigator/NavItem.svelte               | 1 -
 plugins/chunter-resources/src/components/chat/types.ts         | 1 -
 5 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/packages/ui/src/components/NavItem.svelte b/packages/ui/src/components/NavItem.svelte
index e2e0290585..f511ab7bc0 100644
--- a/packages/ui/src/components/NavItem.svelte
+++ b/packages/ui/src/components/NavItem.svelte
@@ -44,7 +44,6 @@
   export let disabled: boolean = false
   export let isFold: boolean = false
   export let isOpen: boolean = false
-  export let isSecondary: boolean = false
   export let withBackground: boolean = false
   export let showMenu: boolean = false
   export let shouldTooltip: boolean = false
@@ -78,7 +77,7 @@
 
 <!-- svelte-ignore a11y-mouse-events-have-key-events -->
 <button
-  class="hulyNavItem-container line-height-auto {type} {type === 'type-anchor-link' || isSecondary
+  class="hulyNavItem-container line-height-auto {type} {type === 'type-anchor-link'
     ? 'font-regular-12'
     : 'font-regular-14'}"
   class:selected
diff --git a/plugins/chunter-resources/src/components/chat/navigator/ChatNavItem.svelte b/plugins/chunter-resources/src/components/chat/navigator/ChatNavItem.svelte
index 1f60ea29a6..8ee6715691 100644
--- a/plugins/chunter-resources/src/components/chat/navigator/ChatNavItem.svelte
+++ b/plugins/chunter-resources/src/components/chat/navigator/ChatNavItem.svelte
@@ -123,7 +123,6 @@
   _id={item.id}
   icon={item.icon}
   withIconBackground={item.withIconBackground}
-  isSecondary={item.isSecondary}
   iconSize={item.iconSize}
   {isSelected}
   iconProps={{ ...item.iconProps, value: item.object }}
diff --git a/plugins/chunter-resources/src/components/chat/navigator/ChatNavSection.svelte b/plugins/chunter-resources/src/components/chat/navigator/ChatNavSection.svelte
index 5e35179cad..9dad5d2f89 100644
--- a/plugins/chunter-resources/src/components/chat/navigator/ChatNavSection.svelte
+++ b/plugins/chunter-resources/src/components/chat/navigator/ChatNavSection.svelte
@@ -91,8 +91,7 @@
           icon: icon ?? getObjectIcon(_class),
           iconProps: { showStatus: true },
           iconSize,
-          withIconBackground: !isDirect && !isPerson,
-          isSecondary: isDocChat && !isPerson
+          withIconBackground: !isDirect && !isPerson
         })
       }
 
diff --git a/plugins/chunter-resources/src/components/chat/navigator/NavItem.svelte b/plugins/chunter-resources/src/components/chat/navigator/NavItem.svelte
index d9aa6e54be..dfc9be5055 100644
--- a/plugins/chunter-resources/src/components/chat/navigator/NavItem.svelte
+++ b/plugins/chunter-resources/src/components/chat/navigator/NavItem.svelte
@@ -71,7 +71,6 @@
   {title}
   {description}
   selected={isSelected}
-  {isSecondary}
   count={elementsCount > 0 ? elementsCount : null}
   {type}
   withBackground={withIconBackground}
diff --git a/plugins/chunter-resources/src/components/chat/types.ts b/plugins/chunter-resources/src/components/chat/types.ts
index df6970ebb9..b2362eae99 100644
--- a/plugins/chunter-resources/src/components/chat/types.ts
+++ b/plugins/chunter-resources/src/components/chat/types.ts
@@ -46,7 +46,6 @@ export interface ChatNavItemModel {
   icon: Asset | AnySvelteComponent | undefined
   iconSize?: IconSize
   iconProps: Record<string, any>
-  isSecondary: boolean
   withIconBackground: boolean
 }
 

From 592688bc79b54705034c8e0d722876eac76bc140 Mon Sep 17 00:00:00 2001
From: Kristina <kristin.fefelova@gmail.com>
Date: Wed, 16 Oct 2024 16:03:02 +0400
Subject: [PATCH 09/12] Improve tabs updates (#6956)

Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com>
---
 .../src/components/Workbench.svelte           |  1 +
 .../components/WorkbenchTabPresenter.svelte   | 20 ++++++++++--
 plugins/workbench-resources/src/workbench.ts  | 31 +++++++++----------
 3 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/plugins/workbench-resources/src/components/Workbench.svelte b/plugins/workbench-resources/src/components/Workbench.svelte
index c5472ee237..cefc3c3078 100644
--- a/plugins/workbench-resources/src/components/Workbench.svelte
+++ b/plugins/workbench-resources/src/components/Workbench.svelte
@@ -221,6 +221,7 @@
           selectTab(tabToReplace._id)
           prevTabIdStore.set(tabToReplace._id)
         } else {
+          console.log('Creating new tab on init')
           const _id = await client.createDoc(workbench.class.WorkbenchTab, core.space.Workspace, {
             attachedTo: account._id,
             location: url,
diff --git a/plugins/workbench-resources/src/components/WorkbenchTabPresenter.svelte b/plugins/workbench-resources/src/components/WorkbenchTabPresenter.svelte
index bcb8d265e3..12dcd0f5f0 100644
--- a/plugins/workbench-resources/src/components/WorkbenchTabPresenter.svelte
+++ b/plugins/workbench-resources/src/components/WorkbenchTabPresenter.svelte
@@ -23,7 +23,7 @@
     languageStore,
     locationToUrl
   } from '@hcengineering/ui'
-  import { ComponentExtensions, getClient } from '@hcengineering/presentation'
+  import { ComponentExtensions, getClient, reduceCalls } from '@hcengineering/presentation'
   import { Asset, getResource, translate } from '@hcengineering/platform'
   import { WorkbenchTab } from '@hcengineering/workbench'
   import view from '@hcengineering/view'
@@ -40,8 +40,18 @@
   let icon: Asset | AnySvelteComponent | undefined
   let iconProps: Record<string, any> | undefined
 
+  let lastLocUrl: string | undefined = undefined
+  let lastLang: string | undefined = undefined
+
   async function updateTabData (tab: WorkbenchTab, lang: string): Promise<void> {
     const tabLoc = $tabIdStore === tab._id ? getCurrentLocation() : getTabLocation(tab)
+    const url = locationToUrl(tabLoc)
+
+    if (lastLocUrl === url && lastLang === lang) return
+
+    lastLocUrl = url
+    lastLang = lang
+
     const data = await getTabDataByLocation(tabLoc)
 
     name = data.name ?? (await translate(data.label, {}, lang))
@@ -53,11 +63,15 @@
     iconProps = data.iconProps
 
     if (tab.name !== name && tab.location === locationToUrl(tabLoc)) {
-      await client.update(tab, { name })
+      await client.diffUpdate(tab, { name })
     }
   }
 
-  $: void updateTabData(tab, $languageStore)
+  const update = reduceCalls(async function (tab: WorkbenchTab, lang: string): Promise<void> {
+    await updateTabData(tab, lang)
+  })
+
+  $: void update(tab, $languageStore)
 
   function handleClickTab (): void {
     selectTab(tab._id)
diff --git a/plugins/workbench-resources/src/workbench.ts b/plugins/workbench-resources/src/workbench.ts
index e15df54854..b7f43cf2a6 100644
--- a/plugins/workbench-resources/src/workbench.ts
+++ b/plugins/workbench-resources/src/workbench.ts
@@ -19,8 +19,8 @@ import core, {
   type Doc,
   getCurrentAccount,
   type Ref,
-  RateLimiter,
-  generateId
+  generateId,
+  reduceCalls
 } from '@hcengineering/core'
 import { type Application, workbenchId, type WorkbenchTab } from '@hcengineering/workbench'
 import {
@@ -56,20 +56,13 @@ tabIdStore.subscribe((value) => {
   prevTabId = value
 })
 
-// Use rate limiter to control tab creation, preventing multiple tabs during fast location changing
-const limiter = new RateLimiter(1)
-
 workspaceStore.subscribe((workspace) => {
   tabIdStore.set(getTabFromLocalStorage(workspace ?? ''))
 })
 
-locationStore.subscribe((l: Location) => {
-  void limiter.add(syncTabLoc)
-})
-
 tabIdStore.subscribe(saveTabToLocalStorage)
 
-async function syncTabLoc (): Promise<void> {
+const syncTabLoc = reduceCalls(async (): Promise<void> => {
   const loc = getCurrentLocation()
   const workspace = get(workspaceStore)
   if (workspace == null || workspace === '') return
@@ -102,7 +95,7 @@ async function syncTabLoc (): Promise<void> {
     }
 
     const me = getCurrentAccount()
-    const tab: WorkbenchTab = {
+    const newTab: WorkbenchTab = {
       _id: generateId(),
       _class: workbench.class.WorkbenchTab,
       space: core.space.Workspace,
@@ -113,9 +106,10 @@ async function syncTabLoc (): Promise<void> {
       modifiedOn: Date.now(),
       modifiedBy: me._id
     }
-    await getClient().createDoc(workbench.class.WorkbenchTab, core.space.Workspace, tab, tab._id)
-    tabsStore.update((tabs) => [...tabs, tab])
-    selectTab(tab._id)
+    console.log('Creating new tab when pinned location changed', { newLocation: url, pinnedLocation: tab.location })
+    await getClient().createDoc(workbench.class.WorkbenchTab, core.space.Workspace, newTab, newTab._id)
+    tabsStore.update((tabs) => [...tabs, newTab])
+    selectTab(newTab._id)
   } else {
     // TODO: Fix this
     // if (
@@ -127,9 +121,14 @@ async function syncTabLoc (): Promise<void> {
     //   return
     // }
 
-    await getClient().update(tab, { location: url, name })
+    await getClient().diffUpdate(tab, { location: url, name })
   }
-}
+})
+
+locationStore.subscribe((l: Location) => {
+  void syncTabLoc()
+})
+
 export function syncWorkbenchTab (): void {
   const workspace = get(workspaceStore)
   tabIdStore.set(getTabFromLocalStorage(workspace ?? ''))

From 3a7b4fecb7dae284d24aa63b7f68fe6470879bfe Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@gmail.com>
Date: Wed, 16 Oct 2024 22:51:10 +0700
Subject: [PATCH 10/12] v0.6.330

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
 common/scripts/version.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/scripts/version.txt b/common/scripts/version.txt
index 3d3b827ee7..2c46cb84e2 100644
--- a/common/scripts/version.txt
+++ b/common/scripts/version.txt
@@ -1 +1 @@
-"0.6.322"
+"0.6.330"

From 9b2cd2c9e95f3bc712ed47f3ac9a4d78ebed9373 Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@users.noreply.github.com>
Date: Wed, 16 Oct 2024 21:14:42 +0700
Subject: [PATCH 11/12] UBERF-8499: Optimize indexer operation (#6959)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
 models/contact/src/index.ts                |   3 +-
 models/controlled-documents/src/index.ts   |   9 +-
 models/core/src/core.ts                    |   6 +-
 models/core/src/index.ts                   |   3 +-
 models/core/src/migration.ts               |   6 +-
 models/gmail/src/index.ts                  |   3 +-
 models/recruit/src/index.ts                |   9 +-
 models/telegram/src/index.ts               |   3 +-
 packages/core/src/classes.ts               |   3 +-
 packages/core/src/component.ts             |   4 +-
 packages/core/src/utils.ts                 |  39 +++++---
 pods/server/package.json                   |   2 +-
 server/elastic/src/adapter.ts              |   2 +-
 server/indexer/src/fulltext.ts             |   4 +-
 server/indexer/src/indexer/field.ts        |  13 +--
 server/indexer/src/indexer/fulltextPush.ts | 105 ++++++++++++++-------
 server/indexer/src/indexer/indexer.ts      |  36 ++++---
 server/indexer/src/indexer/summary.ts      |  49 +++++++---
 server/indexer/src/indexer/types.ts        |  12 +++
 server/indexer/src/indexer/utils.ts        |  63 ++++++-------
 20 files changed, 241 insertions(+), 133 deletions(-)

diff --git a/models/contact/src/index.ts b/models/contact/src/index.ts
index 1109e5549e..e7772d1147 100644
--- a/models/contact/src/index.ts
+++ b/models/contact/src/index.ts
@@ -955,7 +955,8 @@ export function createModel (builder: Builder): void {
   )
 
   // Allow to use fuzzy search for mixins
-  builder.mixin(contact.class.Contact, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: contact.class.Contact,
     fullTextSummary: true
   })
 
diff --git a/models/controlled-documents/src/index.ts b/models/controlled-documents/src/index.ts
index 64145c2ebf..528dfa2fcd 100644
--- a/models/controlled-documents/src/index.ts
+++ b/models/controlled-documents/src/index.ts
@@ -549,7 +549,8 @@ export function createModel (builder: Builder): void {
     func: documents.function.GetAllDocumentStates
   })
 
-  builder.mixin(documents.class.Document, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: documents.class.Document,
     fullTextSummary: true,
     childProcessingAllowed: true
   })
@@ -886,11 +887,13 @@ export function defineNotifications (builder: Builder): void {
 }
 
 export function defineSearch (builder: Builder): void {
-  builder.mixin(documents.class.Document, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: documents.class.Document,
     parentPropagate: true
   })
 
-  builder.mixin(documents.class.DocumentMeta, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: documents.class.DocumentMeta,
     fullTextSummary: true,
     childProcessingAllowed: true,
     propagate: []
diff --git a/models/core/src/core.ts b/models/core/src/core.ts
index b7e6651995..3942868da8 100644
--- a/models/core/src/core.ts
+++ b/models/core/src/core.ts
@@ -361,8 +361,10 @@ export class TDocIndexState extends TDoc implements DocIndexState {
     generationId?: string
 }
 
-@MMixin(core.mixin.FullTextSearchContext, core.class.Class)
-export class TFullTextSearchContext extends TClass implements FullTextSearchContext {}
+@Model(core.class.FullTextSearchContext, core.class.Doc, DOMAIN_MODEL)
+export class TFullTextSearchContext extends TDoc implements FullTextSearchContext {
+  toClass!: Ref<Class<Doc<Space>>>
+}
 
 @MMixin(core.mixin.ConfigurationElement, core.class.Class)
 export class TConfigurationElement extends TClass implements ConfigurationElement {
diff --git a/models/core/src/index.ts b/models/core/src/index.ts
index b80246b6cf..0ef71c7ae9 100644
--- a/models/core/src/index.ts
+++ b/models/core/src/index.ts
@@ -309,7 +309,8 @@ export function createModel (builder: Builder): void {
     ]
   })
 
-  builder.mixin(core.class.Space, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: core.class.Space,
     childProcessingAllowed: false
   })
 
diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts
index 844d1e96f0..5645eec853 100644
--- a/models/core/src/migration.ts
+++ b/models/core/src/migration.ts
@@ -262,7 +262,11 @@ export const coreOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {
     // We need to delete all documents in doc index state for missing classes
     const allClasses = client.hierarchy.getDescendants(core.class.Doc)
-    const allIndexed = allClasses.filter((it) => isClassIndexable(client.hierarchy, it))
+    const contexts = new Map(
+      client.model.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it])
+    )
+
+    const allIndexed = allClasses.filter((it) => isClassIndexable(client.hierarchy, it, contexts))
 
     // Next remove all non indexed classes and missing classes as well.
     await client.update(
diff --git a/models/gmail/src/index.ts b/models/gmail/src/index.ts
index 6e02c0d38c..c7853dcdb4 100644
--- a/models/gmail/src/index.ts
+++ b/models/gmail/src/index.ts
@@ -214,7 +214,8 @@ export function createModel (builder: Builder): void {
     gmail.action.WriteEmail
   )
 
-  builder.mixin(gmail.class.Message, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: gmail.class.Message,
     parentPropagate: false
   })
 
diff --git a/models/recruit/src/index.ts b/models/recruit/src/index.ts
index b1f588e624..5a337a6351 100644
--- a/models/recruit/src/index.ts
+++ b/models/recruit/src/index.ts
@@ -1438,13 +1438,15 @@ export function createModel (builder: Builder): void {
   )
 
   // Allow to use fuzzy search for mixins
-  builder.mixin(recruit.class.Vacancy, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: recruit.class.Vacancy,
     fullTextSummary: true,
     childProcessingAllowed: true,
     propagate: []
   })
 
-  builder.mixin(recruit.mixin.Candidate, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: recruit.mixin.Candidate,
     fullTextSummary: true,
     propagate: [recruit.class.Applicant],
     childProcessingAllowed: true,
@@ -1457,7 +1459,8 @@ export function createModel (builder: Builder): void {
   })
 
   // Allow to use fuzzy search for mixins
-  builder.mixin(recruit.class.Applicant, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: recruit.class.Applicant,
     fullTextSummary: true,
     forceIndex: true,
     childProcessingAllowed: true,
diff --git a/models/telegram/src/index.ts b/models/telegram/src/index.ts
index c950f28c55..80e8a139c4 100644
--- a/models/telegram/src/index.ts
+++ b/models/telegram/src/index.ts
@@ -179,7 +179,8 @@ export function createModel (builder: Builder): void {
     telegram.ids.TelegramMessageSharedActivityViewlet
   )
 
-  builder.mixin(telegram.class.Message, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: telegram.class.Message,
     parentPropagate: false,
     childProcessingAllowed: true
   })
diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts
index 0ce2d198f6..064205df7c 100644
--- a/packages/core/src/classes.ts
+++ b/packages/core/src/classes.ts
@@ -578,7 +578,8 @@ export interface BlobLookup extends Blob {
  *
  * If defined for class, this class will be enabled for embedding search like openai.
  */
-export interface FullTextSearchContext extends Class<Doc> {
+export interface FullTextSearchContext extends Doc {
+  toClass: Ref<Class<Doc>>
   fullTextSummary?: boolean
   forceIndex?: boolean
 
diff --git a/packages/core/src/component.ts b/packages/core/src/component.ts
index 08343ec7f3..5349a3a8c2 100644
--- a/packages/core/src/component.ts
+++ b/packages/core/src/component.ts
@@ -142,10 +142,10 @@ export default plugin(coreId, {
     StatusCategory: '' as Ref<Class<StatusCategory>>,
     MigrationState: '' as Ref<Class<MigrationState>>,
 
-    BenchmarkDoc: '' as Ref<Class<BenchmarkDoc>>
+    BenchmarkDoc: '' as Ref<Class<BenchmarkDoc>>,
+    FullTextSearchContext: '' as Ref<Mixin<FullTextSearchContext>>
   },
   mixin: {
-    FullTextSearchContext: '' as Ref<Mixin<FullTextSearchContext>>,
     ConfigurationElement: '' as Ref<Mixin<ConfigurationElement>>,
     IndexConfiguration: '' as Ref<Mixin<IndexingConfiguration<Doc>>>,
     SpacesTypeData: '' as Ref<Mixin<Space>>
diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts
index f27ed14624..4987f6d617 100644
--- a/packages/core/src/utils.ts
+++ b/packages/core/src/utils.ts
@@ -46,9 +46,9 @@ import core from './component'
 import { Hierarchy } from './hierarchy'
 import { TxOperations } from './operations'
 import { isPredicate } from './predicate'
+import { Branding, BrandingMap } from './server'
 import { DocumentQuery, FindResult } from './storage'
 import { DOMAIN_TX } from './tx'
-import { Branding, BrandingMap } from './server'
 
 function toHex (value: number, chars: number): string {
   const result = value.toString(16)
@@ -686,39 +686,48 @@ export function getFullTextIndexableAttributes (
   return result
 }
 
+const ctxKey = 'indexer_ftc'
 /**
  * @public
  */
 export function getFullTextContext (
   hierarchy: Hierarchy,
-  objectClass: Ref<Class<Doc>>
+  objectClass: Ref<Class<Doc>>,
+  contexts: Map<Ref<Class<Doc>>, FullTextSearchContext>
 ): Omit<FullTextSearchContext, keyof Class<Doc>> {
-  let objClass = hierarchy.getClass(objectClass)
-
-  while (true) {
-    if (hierarchy.hasMixin(objClass, core.mixin.FullTextSearchContext)) {
-      const ctx = hierarchy.as<Class<Doc>, FullTextSearchContext>(objClass, core.mixin.FullTextSearchContext)
+  let ctx: Omit<FullTextSearchContext, keyof Class<Doc>> | undefined = hierarchy.getClassifierProp(objectClass, ctxKey)
+  if (ctx !== undefined) {
+    return ctx
+  }
+  if (typeof ctx !== 'string') {
+    const anc = hierarchy.getAncestors(objectClass)
+    for (const oc of anc) {
+      const ctx = contexts.get(oc)
       if (ctx !== undefined) {
+        hierarchy.setClassifierProp(objectClass, ctxKey, ctx)
         return ctx
       }
     }
-    if (objClass.extends === undefined) {
-      break
-    }
-    objClass = hierarchy.getClass(objClass.extends)
   }
-  return {
+  ctx = {
+    toClass: objectClass,
     fullTextSummary: false,
     forceIndex: false,
     propagate: [],
     childProcessingAllowed: true
   }
+  hierarchy.setClassifierProp(objectClass, ctxKey, ctx)
+  return ctx
 }
 
 /**
  * @public
  */
-export function isClassIndexable (hierarchy: Hierarchy, c: Ref<Class<Doc>>): boolean {
+export function isClassIndexable (
+  hierarchy: Hierarchy,
+  c: Ref<Class<Doc>>,
+  contexts: Map<Ref<Class<Doc>>, FullTextSearchContext>
+): boolean {
   const indexed = hierarchy.getClassifierProp(c, 'class_indexed')
   if (indexed !== undefined) {
     return indexed as boolean
@@ -756,13 +765,13 @@ export function isClassIndexable (hierarchy: Hierarchy, c: Ref<Class<Doc>>): boo
 
   let result = true
 
-  if (attrs.length === 0 && !(getFullTextContext(hierarchy, c)?.forceIndex ?? false)) {
+  if (attrs.length === 0 && !(getFullTextContext(hierarchy, c, contexts)?.forceIndex ?? false)) {
     result = false
     // We need check if document has collections with indexable fields.
     const attrs = hierarchy.getAllAttributes(c).values()
     for (const attr of attrs) {
       if (attr.type._class === core.class.Collection) {
-        if (isClassIndexable(hierarchy, (attr.type as Collection<AttachedDoc>).of)) {
+        if (isClassIndexable(hierarchy, (attr.type as Collection<AttachedDoc>).of, contexts)) {
           result = true
           break
         }
diff --git a/pods/server/package.json b/pods/server/package.json
index f4a303072a..fe0c10e910 100644
--- a/pods/server/package.json
+++ b/pods/server/package.json
@@ -8,7 +8,7 @@
   "template": "@hcengineering/node-package",
   "license": "EPL-2.0",
   "scripts": {
-    "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false MODEL_JSON=../../models/all/bundle/model.json node bundle/bundle.js",
+    "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false MODEL_JSON=../../models/all/bundle/model.json node --inspect bundle/bundle.js",
     "start-u": "rush bundle --to @hcengineering/pod-server && ./bundle/ && cross-env NODE_ENV=production SERVER_PROVIDER=uweb ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json node bundle/bundle.js",
     "start-flame": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json clinic flame --dest ./out -- node --nolazy -r ts-node/register --enable-source-maps src/__start.ts",
     "build": "compile",
diff --git a/server/elastic/src/adapter.ts b/server/elastic/src/adapter.ts
index 2ae65b372c..7b782abf14 100644
--- a/server/elastic/src/adapter.ts
+++ b/server/elastic/src/adapter.ts
@@ -522,7 +522,7 @@ class ElasticAdapter implements FullTextAdapter {
   async updateMany (docs: IndexedDoc[]): Promise<TxResult[]> {
     const parts = Array.from(docs)
     while (parts.length > 0) {
-      const part = parts.splice(0, 1000)
+      const part = parts.splice(0, 500)
 
       const operations = part.flatMap((doc) => {
         const wsDoc = { workspaceId: this.workspaceString, ...doc }
diff --git a/server/indexer/src/fulltext.ts b/server/indexer/src/fulltext.ts
index 1eaa7df8fc..b31805a42c 100644
--- a/server/indexer/src/fulltext.ts
+++ b/server/indexer/src/fulltext.ts
@@ -45,9 +45,9 @@ import core, {
   toFindResult
 } from '@hcengineering/core'
 import type { FullTextAdapter, IndexedDoc, SessionFindAll, StorageAdapter, WithFind } from '@hcengineering/server-core'
-import { getScoringConfig, mapSearchResultDoc } from './mapper'
 import { type FullTextIndexPipeline } from './indexer'
 import { createStateDoc } from './indexer/utils'
+import { getScoringConfig, mapSearchResultDoc } from './mapper'
 
 /**
  * @public
@@ -90,7 +90,7 @@ export class FullTextIndex implements WithFind {
       if (TxProcessor.isExtendsCUD(tx._class)) {
         const cud = tx as TxCUD<Doc>
 
-        if (!isClassIndexable(this.hierarchy, cud.objectClass)) {
+        if (!isClassIndexable(this.hierarchy, cud.objectClass, this.indexer.contexts)) {
           // No need, since no indixable fields or attachments.
           continue
         }
diff --git a/server/indexer/src/indexer/field.ts b/server/indexer/src/indexer/field.ts
index 0e4ce1373b..c2e4914cfc 100644
--- a/server/indexer/src/indexer/field.ts
+++ b/server/indexer/src/indexer/field.ts
@@ -50,6 +50,7 @@ export class IndexedFieldStage implements FullTextPipelineStage {
   updateFields: DocUpdateHandler[] = []
 
   enabled = true
+
   constructor (private readonly dbStorageFindAll: SessionFindAll) {}
 
   async initialize (ctx: MeasureContext, storage: DbAdapter, pipeline: FullTextPipeline): Promise<void> {}
@@ -150,7 +151,7 @@ export class IndexedFieldStage implements FullTextPipelineStage {
           }
 
           if (docState.attachedTo != null && changes > 0) {
-            const ctx = getFullTextContext(pipeline.hierarchy, objClass)
+            const ctx = getFullTextContext(pipeline.hierarchy, objClass, pipeline.contexts)
             if (ctx.parentPropagate ?? true) {
               // We need to clear field stage from parent, so it will be re indexed.
               await pipeline.update(docState.attachedTo as Ref<DocIndexState>, false, {})
@@ -173,17 +174,13 @@ export class IndexedFieldStage implements FullTextPipelineStage {
                     {
                       attachedTo: ids.length === 1 ? ids[0] : { $in: ids }
                     },
-                    { limit: ids.length }
+                    { limit: ids.length, skipSpace: true, skipClass: true }
                   )
               )
             }
             const childs = allChildDocs.filter((it) => it.attachedTo === docState._id)
-            for (const u of childs) {
-              if (propagate.some((it) => pipeline.hierarchy.isDerived(u.objectClass, it))) {
-                pipeline.add(u)
-                await pipeline.update(u._id, false, {})
-              }
-            }
+            // Marck childs to be indexed on next step
+            await pipeline.queue(metrics, new Map(childs.map((it) => [it._id, { updated: true, removed: false }])))
           }
 
           await pipeline.update(docState._id, true, docUpdate)
diff --git a/server/indexer/src/indexer/fulltextPush.ts b/server/indexer/src/indexer/fulltextPush.ts
index 26bf4f9c18..410eec6ed3 100644
--- a/server/indexer/src/indexer/fulltextPush.ts
+++ b/server/indexer/src/indexer/fulltextPush.ts
@@ -30,12 +30,13 @@ import core, {
   type MeasureContext,
   RateLimiter,
   type Ref,
+  SortingOrder,
   toIdMap,
   type WorkspaceId
 } from '@hcengineering/core'
 import { type DbAdapter, type FullTextAdapter, type IndexedDoc, type SessionFindAll } from '@hcengineering/server-core'
-import { updateDocWithPresenter } from '../mapper'
 import { jsonToText, markupToJSON } from '@hcengineering/text'
+import { updateDocWithPresenter } from '../mapper'
 import {
   contentStageId,
   type DocUpdateHandler,
@@ -118,7 +119,7 @@ export class FullTextPushStage implements FullTextPipelineStage {
 
       const childIds = toIndexPart
         .filter((it) => {
-          const fctx = getFullTextContext(pipeline.hierarchy, it.objectClass)
+          const fctx = getFullTextContext(pipeline.hierarchy, it.objectClass, pipeline.contexts)
           return fctx.childProcessingAllowed ?? true
         })
         .map((it) => it._id)
@@ -127,9 +128,17 @@ export class FullTextPushStage implements FullTextPipelineStage {
         'find-child',
         {},
         async (ctx) =>
-          await this.dbStorageFindAll(ctx, core.class.DocIndexState, {
-            attachedTo: childIds.length === 1 ? childIds[0] : { $in: childIds }
-          })
+          await this.dbStorageFindAll(
+            ctx,
+            core.class.DocIndexState,
+            {
+              attachedTo: childIds.length === 1 ? childIds[0] : { $in: childIds }
+            },
+            {
+              skipClass: true,
+              skipSpace: true
+            }
+          )
       )
 
       // spaces
@@ -138,14 +147,22 @@ export class FullTextPushStage implements FullTextPipelineStage {
           'find-spaces',
           {},
           async (ctx) =>
-            await this.dbStorageFindAll(ctx, core.class.DocIndexState, {
-              _id: {
-                $in: toIndexPart.map(
-                  (doc) =>
-                    (doc.attributes[docKey('space', { _class: doc.objectClass })] ?? doc.space) as Ref<DocIndexState>
-                )
+            await this.dbStorageFindAll(
+              ctx,
+              core.class.DocIndexState,
+              {
+                _id: {
+                  $in: toIndexPart.map(
+                    (doc) =>
+                      (doc.attributes[docKey('space', { _class: doc.objectClass })] ?? doc.space) as Ref<DocIndexState>
+                  )
+                }
+              },
+              {
+                skipClass: true,
+                skipSpace: true
               }
-            })
+            )
         )
       )
 
@@ -163,7 +180,7 @@ export class FullTextPushStage implements FullTextPipelineStage {
           const childDocs = allChildDocs.filter((it) => it.attachedTo === doc._id)
           if (childDocs.length > 0) {
             for (const c of childDocs) {
-              const fctx = getFullTextContext(pipeline.hierarchy, c.objectClass)
+              const fctx = getFullTextContext(pipeline.hierarchy, c.objectClass, pipeline.contexts)
               if (fctx.parentPropagate ?? true) {
                 ctx.withSync('updateDoc2Elastic', {}, (ctx) => {
                   updateDoc2Elastic(
@@ -195,7 +212,11 @@ export class FullTextPushStage implements FullTextPipelineStage {
                       {
                         _id: doc.attachedTo as Ref<DocIndexState>
                       },
-                      { limit: 1 }
+                      {
+                        limit: 1,
+                        skipClass: true,
+                        skipSpace: true
+                      }
                     )
                   ).shift()
                 ))
@@ -217,23 +238,43 @@ export class FullTextPushStage implements FullTextPipelineStage {
 
                 const collectClasses = collectPropagateClasses(pipeline, parentDoc.objectClass)
                 if (collectClasses.length > 0) {
-                  const collections = await this.dbStorageFindAll<DocIndexState>(ctx, core.class.DocIndexState, {
-                    attachedTo: parentDoc._id,
-                    objectClass: { $in: collectClasses }
-                  })
-                  for (const c of collections) {
-                    ctx.withSync('updateDoc2Elastic', {}, (ctx) => {
-                      updateDoc2Elastic(
-                        this.allAttrs,
-                        ctx,
-                        c.attributes,
-                        elasticDoc,
-                        c._id,
-                        undefined,
-                        pipeline.hierarchy,
-                        true
-                      )
-                    })
+                  let last: number = 0
+                  while (true) {
+                    const collections = await this.dbStorageFindAll<DocIndexState>(
+                      ctx,
+                      core.class.DocIndexState,
+                      {
+                        attachedTo: parentDoc._id,
+                        objectClass: { $in: collectClasses },
+                        modifiedOn: { $gt: last }
+                      },
+                      {
+                        sort: {
+                          modifiedOn: SortingOrder.Ascending
+                        },
+                        skipClass: true,
+                        skipSpace: true,
+                        limit: 500
+                      }
+                    )
+                    if (collections.length === 0) {
+                      break
+                    }
+                    last = collections[collections.length - 1].modifiedOn
+                    for (const c of collections) {
+                      ctx.withSync('updateDoc2Elastic', {}, (ctx) => {
+                        updateDoc2Elastic(
+                          this.allAttrs,
+                          ctx,
+                          c.attributes,
+                          elasticDoc,
+                          c._id,
+                          undefined,
+                          pipeline.hierarchy,
+                          true
+                        )
+                      })
+                    }
                   }
                 }
               }
@@ -262,7 +303,7 @@ export class FullTextPushStage implements FullTextPipelineStage {
       }
       // Perform bulk update to elastic
 
-      void pushQueue.add(async () => {
+      await pushQueue.exec(async () => {
         try {
           try {
             await ctx.with('push-elastic', {}, async () => {
diff --git a/server/indexer/src/indexer/indexer.ts b/server/indexer/src/indexer/indexer.ts
index c27d7e1baf..9c82cc32e5 100644
--- a/server/indexer/src/indexer/indexer.ts
+++ b/server/indexer/src/indexer/indexer.ts
@@ -21,6 +21,7 @@ import core, {
   type DocIndexState,
   type DocumentQuery,
   type DocumentUpdate,
+  type FullTextSearchContext,
   type Hierarchy,
   type MeasureContext,
   type ModelDb,
@@ -79,6 +80,10 @@ export class FullTextIndexPipeline implements FullTextPipeline {
 
   uploadOps: DocIndexState[] = []
 
+  contexts: Map<Ref<Class<Doc>>, FullTextSearchContext>
+  propogage = new Map<Ref<Class<Doc>>, Ref<Class<Doc>>[]>()
+  propogageClasses = new Map<Ref<Class<Doc>>, Ref<Class<Doc>>[]>()
+
   constructor (
     private readonly storage: DbAdapter,
     private readonly stages: FullTextPipelineStage[],
@@ -90,6 +95,7 @@ export class FullTextIndexPipeline implements FullTextPipeline {
   ) {
     this.readyStages = stages.map((it) => it.stageId)
     this.readyStages.sort()
+    this.contexts = new Map(model.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it]))
   }
 
   async cancel (): Promise<void> {
@@ -386,8 +392,6 @@ export class FullTextIndexPipeline implements FullTextPipeline {
       )
 
       // Also update doc index state queries.
-      _classes.push(core.class.DocIndexState)
-
       _classes.forEach((it) => this.broadcastClasses.add(it))
 
       if (this.triggerCounts > 0) {
@@ -410,10 +414,16 @@ export class FullTextIndexPipeline implements FullTextPipeline {
             }
           }, 5000)
 
+          let notified = false
           await new Promise((resolve) => {
             this.triggerIndexing = () => {
               this.triggerCounts++
-              resolve(null)
+              if (!notified) {
+                notified = true
+                setTimeout(() => {
+                  resolve(null)
+                }, 500) // Start indexing only after cooldown
+              }
             }
           })
         }
@@ -435,14 +445,18 @@ export class FullTextIndexPipeline implements FullTextPipeline {
           })
 
           let result: DocIndexState[] | undefined = await ctx.with('get-indexable', {}, async () => {
-            const q: DocumentQuery<DocIndexState> = {
-              needIndex: true
-            }
-            return await this.storage.findAll(ctx, core.class.DocIndexState, q, {
-              limit: globalIndexer.processingSize,
-              skipClass: true,
-              skipSpace: true
-            })
+            return await this.storage.findAll(
+              ctx,
+              core.class.DocIndexState,
+              {
+                needIndex: true
+              },
+              {
+                limit: globalIndexer.processingSize,
+                skipClass: true,
+                skipSpace: true
+              }
+            )
           })
           if (result === undefined) {
             // No more results
diff --git a/server/indexer/src/indexer/summary.ts b/server/indexer/src/indexer/summary.ts
index 6a6212317d..56096ab8b9 100644
--- a/server/indexer/src/indexer/summary.ts
+++ b/server/indexer/src/indexer/summary.ts
@@ -114,7 +114,7 @@ export class FullSummaryStage implements FullTextPipelineStage {
         const childDocs = allChildDocs.filter((it) => it.attachedTo === doc._id)
         if (childDocs.length > 0) {
           for (const c of childDocs) {
-            const ctx = getFullTextContext(pipeline.hierarchy, c.objectClass)
+            const ctx = getFullTextContext(pipeline.hierarchy, c.objectClass, pipeline.contexts)
             if (ctx.parentPropagate ?? true) {
               if (embeddingText.length > this.summaryLimit) {
                 break
@@ -137,22 +137,43 @@ export class FullSummaryStage implements FullTextPipelineStage {
               metrics,
               core.class.DocIndexState,
               { _id: doc.attachedTo as Ref<DocIndexState> },
-              { limit: 1 }
+              {
+                limit: 1,
+                skipSpace: true,
+                skipClass: true
+              }
             )
             if (parentDoc !== undefined) {
               const ctx = collectPropagateClasses(pipeline, parentDoc.objectClass)
               if (ctx.length > 0) {
-                const collections = await this.dbStorageFindAll(metrics, core.class.DocIndexState, {
-                  attachedTo: parentDoc._id,
-                  objectClass: ctx.length === 1 ? ctx[0] : { $in: ctx }
-                })
-                for (const c of collections) {
-                  embeddingText +=
-                    '\n' +
-                    (await extractIndexedValues(c, pipeline.hierarchy, {
-                      matchExtra: this.matchExtra,
-                      fieldFilter: this.fieldFilter
-                    }))
+                let last = 0
+                while (true) {
+                  const collections = await this.dbStorageFindAll(
+                    metrics,
+                    core.class.DocIndexState,
+                    {
+                      attachedTo: parentDoc._id,
+                      objectClass: ctx.length === 1 ? ctx[0] : { $in: ctx },
+                      modifiedOn: { $gt: last }
+                    },
+                    {
+                      limit: 250,
+                      skipClass: true,
+                      skipSpace: true
+                    }
+                  )
+                  if (collections.length === 0) {
+                    break
+                  }
+                  last = collections[collections.length - 1].modifiedOn
+                  for (const c of collections) {
+                    embeddingText +=
+                      '\n' +
+                      (await extractIndexedValues(c, pipeline.hierarchy, {
+                        matchExtra: this.matchExtra,
+                        fieldFilter: this.fieldFilter
+                      }))
+                  }
                 }
               }
 
@@ -188,7 +209,7 @@ export class FullSummaryStage implements FullTextPipelineStage {
  * @public
  */
 export function isIndexingRequired (pipeline: FullTextPipeline, doc: DocIndexState): boolean {
-  return getFullTextContext(pipeline.hierarchy, doc.objectClass).fullTextSummary ?? false
+  return getFullTextContext(pipeline.hierarchy, doc.objectClass, pipeline.contexts).fullTextSummary ?? false
 }
 
 /**
diff --git a/server/indexer/src/indexer/types.ts b/server/indexer/src/indexer/types.ts
index e55b2dc4f1..91bec382a4 100644
--- a/server/indexer/src/indexer/types.ts
+++ b/server/indexer/src/indexer/types.ts
@@ -19,6 +19,7 @@ import {
   type DocIndexState,
   type DocumentQuery,
   type DocumentUpdate,
+  type FullTextSearchContext,
   type Hierarchy,
   type MeasureContext,
   type ModelDb,
@@ -32,6 +33,12 @@ import type { DbAdapter, IndexedDoc } from '@hcengineering/server-core'
 export interface FullTextPipeline {
   hierarchy: Hierarchy
   model: ModelDb
+
+  contexts: Map<Ref<Class<Doc>>, FullTextSearchContext>
+
+  propogage: Map<Ref<Class<Doc>>, Ref<Class<Doc>>[]>
+  propogageClasses: Map<Ref<Class<Doc>>, Ref<Class<Doc>>[]>
+
   update: (
     docId: Ref<DocIndexState>,
     mark: boolean,
@@ -49,6 +56,11 @@ export interface FullTextPipeline {
     from?: number
   ) => Promise<{ docs: IndexedDoc[], pass: boolean }>
 
+  queue: (
+    ctx: MeasureContext,
+    updates: Map<Ref<DocIndexState>, { create?: DocIndexState, updated: boolean, removed: boolean }>
+  ) => Promise<void>
+
   cancelling: boolean
 }
 
diff --git a/server/indexer/src/indexer/utils.ts b/server/indexer/src/indexer/utils.ts
index 8ca1d1ab38..2f1e57bf25 100644
--- a/server/indexer/src/indexer/utils.ts
+++ b/server/indexer/src/indexer/utils.ts
@@ -82,58 +82,55 @@ export function traverseFullTextContexts (
   pipeline: FullTextPipeline,
   objectClass: Ref<Class<Doc>>,
   op: (ftc: Omit<FullTextSearchContext, keyof Class<Doc>>) => void
-): Ref<Class<Doc>>[] {
-  const desc = new Set(pipeline.hierarchy.getDescendants(objectClass))
-  const propagate = new Set<Ref<Class<Doc>>>()
-
-  const ftContext = getFullTextContext(pipeline.hierarchy, objectClass)
+): void {
+  const cl = pipeline.hierarchy.getBaseClass(objectClass)
+  const ftContext = getFullTextContext(pipeline.hierarchy, cl, pipeline.contexts)
   if (ftContext !== undefined) {
     op(ftContext)
   }
-
-  // Add all parent mixins as well
-  for (const a of pipeline.hierarchy.getAncestors(objectClass)) {
-    const ftContext = getFullTextContext(pipeline.hierarchy, a)
-    if (ftContext !== undefined) {
-      op(ftContext)
-    }
-    const dsca = pipeline.hierarchy.getDescendants(a)
-    for (const dd of dsca) {
-      if (pipeline.hierarchy.isMixin(dd)) {
-        desc.add(dd)
-      }
+  const dsca = pipeline.hierarchy.getDescendants(cl)
+  for (const dd of dsca) {
+    const mContext = getFullTextContext(pipeline.hierarchy, dd, pipeline.contexts)
+    if (mContext !== undefined) {
+      op(mContext)
     }
   }
-
-  for (const d of desc) {
-    if (pipeline.hierarchy.isMixin(d)) {
-      const mContext = getFullTextContext(pipeline.hierarchy, d)
-      if (mContext !== undefined) {
-        op(mContext)
-      }
-    }
-  }
-  return Array.from(propagate.values())
 }
 
 /**
  * @public
  */
 export function collectPropagate (pipeline: FullTextPipeline, objectClass: Ref<Class<Doc>>): Ref<Class<Doc>>[] {
-  const propagate = new Set<Ref<Class<Doc>>>()
-  traverseFullTextContexts(pipeline, objectClass, (fts) => fts?.propagate?.forEach((it) => propagate.add(it)))
+  let propagate = pipeline.propogage.get(objectClass)
+  if (propagate !== undefined) {
+    return propagate
+  }
+  const set = new Set<Ref<Class<Doc>>>()
+  traverseFullTextContexts(pipeline, objectClass, (fts) => {
+    fts?.propagate?.forEach((it) => {
+      set.add(it)
+    })
+  })
 
-  return Array.from(propagate.values())
+  propagate = Array.from(set.values())
+  pipeline.propogage.set(objectClass, propagate)
+  return propagate
 }
 
 /**
  * @public
  */
 export function collectPropagateClasses (pipeline: FullTextPipeline, objectClass: Ref<Class<Doc>>): Ref<Class<Doc>>[] {
-  const propagate = new Set<Ref<Class<Doc>>>()
-  traverseFullTextContexts(pipeline, objectClass, (fts) => fts?.propagateClasses?.forEach((it) => propagate.add(it)))
+  let propagate = pipeline.propogageClasses.get(objectClass)
+  if (propagate !== undefined) {
+    return propagate
+  }
+  const set = new Set<Ref<Class<Doc>>>()
+  traverseFullTextContexts(pipeline, objectClass, (fts) => fts?.propagateClasses?.forEach((it) => set.add(it)))
 
-  return Array.from(propagate.values())
+  propagate = Array.from(set.values())
+  pipeline.propogageClasses.set(objectClass, propagate)
+  return propagate
 }
 
 const CUSTOM_ATTR_KEY = 'customAttributes'

From 994229df9e4191440a950d03195e7500dfac466e Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@gmail.com>
Date: Wed, 16 Oct 2024 23:41:12 +0700
Subject: [PATCH 12/12] v0.6.331

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
 common/scripts/version.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/scripts/version.txt b/common/scripts/version.txt
index 2c46cb84e2..38a6745cb3 100644
--- a/common/scripts/version.txt
+++ b/common/scripts/version.txt
@@ -1 +1 @@
-"0.6.330"
+"0.6.331"