Merge remote-tracking branch 'origin/develop' into staging

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

models/ai-bot/package.json

@@ -31,8 +31,10 @@
     "@hcengineering/ai-bot": "^0.6.0",
     "@hcengineering/analytics-collector": "^0.6.0",
     "@hcengineering/chunter": "^0.6.20",
+    "@hcengineering/contact": "^0.6.24",
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/model": "^0.6.11",
+    "@hcengineering/model-contact": "^0.6.1",
     "@hcengineering/model-core": "^0.6.0",
     "@hcengineering/model-view": "^0.6.0",
     "@hcengineering/platform": "^0.6.11",

models/ai-bot/src/index.ts

@@ -21,6 +21,7 @@ import analyticsCollector from '@hcengineering/analytics-collector'
 import aiBot from './plugin'
 
 export { aiBotId } from '@hcengineering/ai-bot'
+export { aiBotOperation } from './migration'
 export default aiBot
 
 export const DOMAIN_AI_BOT = 'ai_bot' as Domain

models/ai-bot/src/migration.ts

@@ -0,0 +1,86 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import contact, { type Channel, type Person, type PersonAccount } from '@hcengineering/contact'
+import core, {
+  DOMAIN_MODEL_TX,
+  type TxCUD,
+  type TxCreateDoc,
+  type Ref,
+  type TxUpdateDoc,
+  TxProcessor,
+  type Domain
+} from '@hcengineering/core'
+import { DOMAIN_CHANNEL, DOMAIN_CONTACT } from '@hcengineering/model-contact'
+import {
+  tryMigrate,
+  type MigrateOperation,
+  type MigrationClient,
+  type MigrationUpgradeClient
+} from '@hcengineering/model'
+import aiBot, { aiBotId } from '@hcengineering/ai-bot'
+
+const DOMAIN_ACTIVITY = 'activity' as Domain
+
+async function migrateAiExtraAccounts (client: MigrationClient): Promise<void> {
+  const currentAccount = (
+    await client.model.findAll(contact.class.PersonAccount, { _id: aiBot.account.AIBot as Ref<PersonAccount> })
+  )[0]
+  if (currentAccount === undefined) return
+
+  const txes = await client.find<TxCUD<PersonAccount>>(DOMAIN_MODEL_TX, {
+    _class: { $in: [core.class.TxCreateDoc, core.class.TxUpdateDoc] },
+    objectClass: contact.class.PersonAccount,
+    objectId: aiBot.account.AIBot as Ref<PersonAccount>
+  })
+
+  const personsToDelete: Ref<Person>[] = []
+  const txesToDelete: Ref<TxCUD<PersonAccount>>[] = []
+
+  for (const tx of txes) {
+    if (tx._class === core.class.TxCreateDoc) {
+      const acc = TxProcessor.createDoc2Doc(tx as TxCreateDoc<PersonAccount>)
+      if (acc.person !== currentAccount.person) {
+        personsToDelete.push(acc.person)
+        txesToDelete.push(tx._id)
+      }
+    } else if (tx._class === core.class.TxUpdateDoc) {
+      const person = (tx as TxUpdateDoc<PersonAccount>).operations.person
+      if (person !== undefined && person !== currentAccount.person) {
+        personsToDelete.push(person)
+        txesToDelete.push(tx._id)
+      }
+    }
+  }
+
+  if (personsToDelete.length === 0) return
+
+  await client.deleteMany(DOMAIN_MODEL_TX, { _id: { $in: txesToDelete } })
+  await client.deleteMany(DOMAIN_ACTIVITY, { attachedTo: { $in: personsToDelete } })
+  await client.deleteMany<Channel>(DOMAIN_CHANNEL, { attachedTo: { $in: personsToDelete } })
+  await client.deleteMany(DOMAIN_CONTACT, { _id: { $in: personsToDelete } })
+}
+
+export const aiBotOperation: MigrateOperation = {
+  async migrate (client: MigrationClient): Promise<void> {
+    await tryMigrate(client, aiBotId, [
+      {
+        state: 'remove-ai-bot-extra-accounts-v100',
+        func: migrateAiExtraAccounts
+      }
+    ])
+  },
+  async upgrade (state: Map<string, Set<string>>, client: () => Promise<MigrationUpgradeClient>): Promise<void> {}
+}

models/all/src/migration.ts

@@ -54,6 +54,7 @@ import { analyticsCollectorOperation } from '@hcengineering/model-analytics-coll
 import { workbenchOperation } from '@hcengineering/model-workbench'
 import { testManagementOperation } from '@hcengineering/model-test-management'
 import { surveyOperation } from '@hcengineering/model-survey'
+import { aiBotId, aiBotOperation } from '@hcengineering/model-ai-bot'
 
 export const migrateOperations: [string, MigrateOperation][] = [
   ['core', coreOperation],
@@ -96,5 +97,6 @@ export const migrateOperations: [string, MigrateOperation][] = [
   ['analyticsCollector', analyticsCollectorOperation],
   ['workbench', workbenchOperation],
   ['testManagement', testManagementOperation],
-  ['survey', surveyOperation]
+  ['survey', surveyOperation],
+  [aiBotId, aiBotOperation]
 ]

models/recruit/src/index.ts

@@ -355,7 +355,7 @@ export function createModel (builder: Builder): void {
         }
       ],
       configOptions: {
-        hiddenKeys: ['name', 'space', 'modifiedOn'],
+        hiddenKeys: ['name', 'space', 'modifiedOn', 'company'],
         sortable: true
       },
       viewOptions: {

plugins/login-assets/lang/cs.json

@@ -19,6 +19,7 @@
     "DoNotHaveAnAccount": "Nemáte účet?",
     "PasswordRepeat": "Zopakujte heslo",
     "HaveAccount": "Již máte účet?",
+    "LoadingAccount": "Načítání...",
     "SelectWorkspace": "Vyberte pracovní prostor",
     "Copy": "Kopírovat",
     "Copied": "Zkopírováno",

plugins/login-assets/lang/en.json

@@ -19,6 +19,7 @@
     "DoNotHaveAnAccount": "Do not have an account?",
     "PasswordRepeat": "Repeat password",
     "HaveAccount": "Already have an account?",
+    "LoadingAccount": "Loading...",
     "SelectWorkspace": "Select workspace",
     "Copy": "Copy",
     "Copied": "Copied",

plugins/login-assets/lang/es.json

@@ -19,6 +19,7 @@
     "DoNotHaveAnAccount": "¿No tienes una cuenta?",
     "PasswordRepeat": "Repetir contraseña",
     "HaveAccount": "¿Ya tienes una cuenta?",
+    "LoadingAccount": "Cargando...",
     "SelectWorkspace": "Seleccionar espacio de trabajo",
     "Copy": "Copiar",
     "Copied": "Copiado",

plugins/login-assets/lang/fr.json

@@ -19,6 +19,7 @@
     "DoNotHaveAnAccount": "Vous n'avez pas de compte ?",
     "PasswordRepeat": "Répétez le mot de passe",
     "HaveAccount": "Vous avez déjà un compte ?",
+    "LoadingAccount": "Chargement...",
     "SelectWorkspace": "Sélectionner un espace de travail",
     "Copy": "Copier",
     "Copied": "Copié",

plugins/login-assets/lang/it.json

@@ -19,6 +19,7 @@
     "DoNotHaveAnAccount": "Non hai un account?",
     "PasswordRepeat": "Ripeti password",
     "HaveAccount": "Hai già un account?",
+    "LoadingAccount": "Caricamento...",
     "SelectWorkspace": "Seleziona spazio di lavoro",
     "Copy": "Copia",
     "Copied": "Copiato",

plugins/login-assets/lang/pt.json

@@ -19,6 +19,7 @@
     "DoNotHaveAnAccount": "Não tem uma conta?",
     "PasswordRepeat": "Repetir palavra-passe",
     "HaveAccount": "Já tem uma conta?",
+    "LoadingAccount": "Carregando...",
     "SelectWorkspace": "Selecionar espaço de trabalho",
     "Copy": "Copiar",
     "Copied": "Copiado",

plugins/login-assets/lang/ru.json

@@ -19,6 +19,7 @@
     "DoNotHaveAnAccount": "Нет учетной записи?",
     "PasswordRepeat": "Повторите пароль",
     "HaveAccount": "Уже есть учетная запись?",
+    "LoadingAccount": "Загрузка...",
     "SelectWorkspace": "Выбрать рабочее пространство",
     "Copy": "Копировать",
     "Copied": "Скопировано",

plugins/login-assets/lang/zh.json

@@ -19,6 +19,7 @@
     "DoNotHaveAnAccount": "还没有账户？",
     "PasswordRepeat": "重复密码",
     "HaveAccount": "已经有账户了？",
+    "LoadingAccount": "加载中...",
     "SelectWorkspace": "选择工作区",
     "Copy": "复制",
     "Copied": "已复制",

plugins/login-resources/src/components/SelectWorkspace.svelte

@@ -20,6 +20,7 @@
   import {
     Button,
     Label,
+    Spinner,
     Scroller,
     SearchEdit,
     deviceOptionsStore as deviceInfo,
@@ -95,7 +96,11 @@
 <form class="container" style:padding={$deviceInfo.docWidth <= 480 ? '1.25rem' : '5rem'}>
   <div class="grow-separator" />
   <div class="fs-title">
-    {account?.email}
+    {#if account?.email}
+      {account.email}
+    {:else}
+      <Label label={login.string.LoadingAccount} />
+    {/if}
   </div>
   <div class="title"><Label label={login.string.SelectWorkspace} /></div>
   <div class="status">
@@ -106,7 +111,11 @@
       <SearchEdit bind:value={search} width={'100%'} />
     </div>
   {/if}
-  {#await _getWorkspaces() then}
+  {#await _getWorkspaces()}
+    <div class="workspace-loader">
+      <Spinner />
+    </div>
+  {:then}
     <Scroller padding={'.125rem 0'} maxHeight={35}>
       <div class="form">
         {#each workspaces
@@ -207,6 +216,13 @@
     flex-grow: 1;
     overflow: hidden;
 
+    .workspace-loader {
+      height: 100%;
+      display: flex;
+      justify-content: center;
+      align-items: center;
+    }
+
     .title {
       font-weight: 600;
       font-size: 1.5rem;

plugins/login-resources/src/plugin.ts

@@ -32,6 +32,7 @@ export default mergeIds(loginId, login, {
     LastName: '' as IntlString,
     FirstName: '' as IntlString,
     HaveAccount: '' as IntlString,
+    LoadingAccount: '' as IntlString,
     Join: '' as IntlString,
     Email: '' as IntlString,
     Password: '' as IntlString,

plugins/recruit-resources/src/components/EditVacancy.svelte

@@ -208,7 +208,15 @@
       <VacancyApplications objectId={object._id} {readonly} />
     </div>
     <div class="w-full mt-6">
-      <Component is={survey.component.PollCollection} props={{ object, label: survey.string.Polls }} />
+      <Component
+        is={survey.component.PollCollection}
+        props={{
+          objectId: object._id,
+          _class: object._class,
+          space: object.space,
+          polls: object.polls
+        }}
+      />
     </div>
     <div class="w-full mt-6">
       <Component is={tracker.component.RelatedIssuesSection} props={{ object, label: tracker.string.RelatedIssues }} />

plugins/survey-resources/src/components/PollCollection.svelte

@@ -47,7 +47,7 @@
     const client = getClient()
     const pollId = await client.addCollection(survey.class.Poll, space, objectId, _class, 'polls', makePollData(source))
 
-    const poll = await client.findOne(survey.class.Survey, { _id: pollId })
+    const poll = await client.findOne(survey.class.Poll, { _id: pollId })
     if (poll === undefined) {
       console.error(`Could not find just created poll ${pollId}.`)
       return

services/ai-bot/pod-ai-bot/src/controller.ts

@@ -51,7 +51,7 @@ const CLOSE_INTERVAL_MS = 10 * 60 * 1000 // 10 minutes
 export class AIControl {
   private readonly workspaces: Map<string, WorkspaceClient> = new Map<string, WorkspaceClient>()
   private readonly closeWorkspaceTimeouts: Map<string, NodeJS.Timeout> = new Map<string, NodeJS.Timeout>()
-  private readonly connectingWorkspaces: Set<string> = new Set<string>()
+  private readonly connectingWorkspaces = new Map<string, Promise<void>>()
 
   readonly aiClient?: OpenAI
   readonly encoding = encodingForModel(config.OpenAIModel)
@@ -69,7 +69,6 @@ export class AIControl {
           baseURL: config.OpenAIBaseUrl === '' ? undefined : config.OpenAIBaseUrl
         })
         : undefined
-
     void this.connectSupportWorkspace()
   }
 
@@ -78,11 +77,9 @@ export class AIControl {
   }
 
   async connectSupportWorkspace (): Promise<void> {
-    if (this.supportClient === undefined && !this.connectingWorkspaces.has(config.SupportWorkspace)) {
+    if (this.supportClient === undefined) {
-      this.connectingWorkspaces.add(config.SupportWorkspace)
       const record = await this.getWorkspaceRecord(config.SupportWorkspace)
       this.supportClient = (await this.createWorkspaceClient(config.SupportWorkspace, record)) as SupportWsClient
-      this.connectingWorkspaces.delete(config.SupportWorkspace)
     }
   }
 
@@ -138,27 +135,36 @@ export class AIControl {
     if (workspace === config.SupportWorkspace) {
       return
     }
-    this.connectingWorkspaces.add(workspace)
 
-    if (!this.workspaces.has(workspace)) {
+    if (this.connectingWorkspaces.has(workspace)) {
-      const record = await this.getWorkspaceRecord(workspace)
+      return await this.connectingWorkspaces.get(workspace)
-      const client = await this.createWorkspaceClient(workspace, record)
+    }
-      if (client === undefined) {
+
+    const initPromise = (async () => {
+      try {
+        if (!this.workspaces.has(workspace)) {
+          const record = await this.getWorkspaceRecord(workspace)
+          const client = await this.createWorkspaceClient(workspace, record)
+          if (client === undefined) {
+            return
+          }
+          this.workspaces.set(workspace, client)
+        }
+
+        const timeoutId = this.closeWorkspaceTimeouts.get(workspace)
+        if (timeoutId !== undefined) {
+          clearTimeout(timeoutId)
+        }
+
+        this.updateClearInterval(workspace)
+      } finally {
         this.connectingWorkspaces.delete(workspace)
-        return
       }
+    })()
 
-      this.workspaces.set(workspace, client)
+    this.connectingWorkspaces.set(workspace, initPromise)
-    }
 
-    const timeoutId = this.closeWorkspaceTimeouts.get(workspace)
+    await initPromise
-
-    if (timeoutId !== undefined) {
-      clearTimeout(timeoutId)
-    }
-
-    this.updateClearInterval(workspace)
-    this.connectingWorkspaces.delete(workspace)
   }
 
   allowAiReplies (workspace: string, email: string): boolean {

services/github/pod-github/src/worker.ts

@@ -24,6 +24,7 @@ import core, {
   TxApplyIf,
   TxCUD,
   TxOperations,
+  TxProcessor,
   TxWorkspaceEvent,
   WithLookup,
   WorkspaceEvent,
@@ -824,12 +825,16 @@ export class GithubWorker implements IntegrationManager {
           // Handle tx
           const h = this._client.getHierarchy()
           for (const t of tx) {
-            if (h.isDerived(t._class, core.class.TxCUD)) {
+            if (TxProcessor.isExtendsCUD(t._class)) {
               const cud = t as TxCUD<Doc>
               if (cud.objectClass === github.class.DocSyncInfo) {
                 this.triggerSync()
                 break
               }
+              if (cud.objectClass === contact.class.Person || cud.objectClass === contact.class.Channel) {
+                this.accountMap.clear()
+                break
+              }
             }
             if (h.isDerived(t._class, core.class.TxApplyIf)) {
               const applyop = t as TxApplyIf

workers/datalake/src/blob.ts

@@ -44,9 +44,13 @@ export async function handleBlobGet (
   const { workspace, name } = request
 
   const cache = new LoggedCache(caches.default, metrics)
-  const cached = await cache.match(request)
+
-  if (cached !== undefined) {
+  const cacheControl = request.headers.get('Cache-Control') ?? ''
-    return cached
+  if (!cacheControl.includes('no-cache')) {
+    const cached = await cache.match(request)
+    if (cached !== undefined) {
+      return cached
+    }
   }
 
   const { bucket } = selectStorage(env, workspace)
@@ -75,8 +79,10 @@ export async function handleBlobGet (
   const response = new Response(object?.body, { headers, status })
 
   if (response.status === 200) {
-    const clone = metrics.withSync('response.clone', () => response.clone())
+    if (!cacheControl.includes('no-store')) {
-    ctx.waitUntil(cache.put(request, clone))
+      const clone = metrics.withSync('response.clone', () => response.clone())
+      ctx.waitUntil(cache.put(request, clone))
+    }
   }
 
   return response

workers/datalake/src/db.ts

@@ -50,12 +50,14 @@ export async function withPostgres<T> (
   fn: (db: BlobDB) => Promise<T>
 ): Promise<T> {
   const sql = metrics.withSync('db.connect', () => {
-    return postgres(env.HYPERDRIVE.connectionString, {
+    return postgres(env.DB_URL, {
       connection: {
         application_name: 'datalake'
-      }
+      },
+      fetch_types: false
     })
   })
+
   const db = new LoggedDB(new PostgresDB(sql), metrics)
 
   try {

workers/datalake/worker-configuration.d.ts

@@ -12,6 +12,7 @@ interface Env {
   STREAMS_ACCOUNT_ID: string;
   STREAMS_AUTH_KEY: string;
   R2_ACCOUNT_ID: string;
+  DB_URL: string;
   DATALAKE_APAC_ACCESS_KEY: string;
   DATALAKE_APAC_SECRET_KEY: string;
   DATALAKE_APAC_BUCKET_NAME: string;