UBERF-6626: More detailed info about maintenance (#5400)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

common/config/rush/pnpm-lock.yaml

@@ -29,6 +29,9 @@ dependencies:
   '@rush-temp/account':
     specifier: file:./projects/account.tgz
     version: file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)
+  '@rush-temp/account-service':
+    specifier: file:./projects/account-service.tgz
+    version: file:projects/account-service.tgz
   '@rush-temp/activity':
     specifier: file:./projects/activity.tgz
     version: file:projects/activity.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2)
@@ -17040,6 +17043,54 @@ packages:
       css-what: 6.1.0
     dev: false
 
+  file:projects/account-service.tgz:
+    resolution: {integrity: sha512-nobaJJXk2cwnaGafGa8skzZZ4Y3tn6AfYwO3SMo0hMyh5VSL6qzpqPo0Qyaf8PEgk/0IhyEt9fY9hqPzGnozAg==, tarball: file:projects/account-service.tgz}
+    name: '@rush-temp/account-service'
+    version: 0.0.0
+    dependencies:
+      '@koa/cors': 3.4.3
+      '@types/jest': 29.5.12
+      '@types/koa': 2.14.0
+      '@types/koa-bodyparser': 4.3.12
+      '@types/koa-router': 7.4.8
+      '@types/koa__cors': 3.3.1
+      '@types/node': 20.11.19
+      '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
+      '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
+      cross-env: 7.0.3
+      esbuild: 0.20.1
+      eslint: 8.56.0
+      eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3)
+      eslint-plugin-import: 2.29.1(eslint@8.56.0)
+      eslint-plugin-n: 15.7.0(eslint@8.56.0)
+      eslint-plugin-promise: 6.1.1(eslint@8.56.0)
+      jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
+      koa: 2.15.0
+      koa-bodyparser: 4.4.1
+      koa-router: 12.0.1
+      mongodb: 6.3.0
+      prettier: 3.2.5
+      ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
+      ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
+      typescript: 5.3.3
+    transitivePeerDependencies:
+      - '@aws-sdk/credential-providers'
+      - '@babel/core'
+      - '@jest/types'
+      - '@mongodb-js/zstd'
+      - '@swc/core'
+      - '@swc/wasm'
+      - babel-jest
+      - babel-plugin-macros
+      - gcp-metadata
+      - kerberos
+      - mongodb-client-encryption
+      - node-notifier
+      - snappy
+      - socks
+      - supports-color
+    dev: false
+
   file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2):
     resolution: {integrity: sha512-9RuPhqNNHTYjQwezirzcJ6WZpMJTzbjc72jZSJC6dQFG7WqW0a2QZ8VTaa32IM902stPP950kaRUDGGEKlxEwg==, tarball: file:projects/account.tgz}
     id: file:projects/account.tgz
@@ -17118,7 +17169,7 @@ packages:
     dev: false
 
   file:projects/activity-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2):
-    resolution: {integrity: sha512-VhX3SSgQDRNCtvHGWeYtqp+OeXxMpwNPv4PxmSS+LukRclxu9XPBBqIPfm8zOAIP+T+dtorTxAJBdlmlHT5kkw==, tarball: file:projects/activity-resources.tgz}
+    resolution: {integrity: sha512-CWMQG4ARK+tkN2nxFphlOBnSzy4afqvMbPHlBC1rEQNaLc+5C7a6s5DMhHfjfd+0PMMGpl3F9+HSIHSKhZF7JQ==, tarball: file:projects/activity-resources.tgz}
     id: file:projects/activity-resources.tgz
     name: '@rush-temp/activity-resources'
     version: 0.0.0
@@ -17420,7 +17471,7 @@ packages:
     dev: false
 
   file:projects/backup-service.tgz(esbuild@0.20.1)(ts-node@10.9.2):
-    resolution: {integrity: sha512-V3tol7QGRHUEOi2hp9fv+nnjYVJJiMo/Pvfl5aRoL/CySk9vq/8KdQ6dBI2cFTLsRVjHiS4F4tzgsxcgZ09DQw==, tarball: file:projects/backup-service.tgz}
+    resolution: {integrity: sha512-tMU5TaEJAhjWjFJZsV2Vx/PUPFL7UIVy4SrVGzuYJvXD+5rAbfeVw2HjGVwazFmmPfTnvD8qwSxsL0od2FjJ2A==, tarball: file:projects/backup-service.tgz}
     id: file:projects/backup-service.tgz
     name: '@rush-temp/backup-service'
     version: 0.0.0
@@ -20661,7 +20712,7 @@ packages:
     dev: false
 
   file:projects/notification-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2):
-    resolution: {integrity: sha512-FQo/OscXLnjmD9vm49ZL+pjyTVIFugcyHORnJwA124EWnWlxP1zezEvopvLqMd2wu6tLzfvhrVqnIEm8HkDfUg==, tarball: file:projects/notification-resources.tgz}
+    resolution: {integrity: sha512-D1CL9lg5xd7hZJxBvzOd3SwHTPH4Up4mREwFPvf1v5GyQUiKqZMW/RKCop1G81hwh4pje5De7cz/TBxa5bc24Q==, tarball: file:projects/notification-resources.tgz}
     id: file:projects/notification-resources.tgz
     name: '@rush-temp/notification-resources'
     version: 0.0.0
@@ -20873,7 +20924,7 @@ packages:
     dev: false
 
   file:projects/pod-account.tgz:
-    resolution: {integrity: sha512-nDD+/VgDS0KrGvPf9CH9ln72wJYfnlAuo6hkcfV8/Mj9vX2S434s62TH/MzgAtepyf7UcIgo5zPaEA9jAMJwtw==, tarball: file:projects/pod-account.tgz}
+    resolution: {integrity: sha512-yjd/f6z0VELT7H3hINe53TvADlDoTwoqC5UJduaQCpiQDB38Tf28WJ0tJWKZc23iGX6cKgUyP1NrmaNPiT9TUw==, tarball: file:projects/pod-account.tgz}
     name: '@rush-temp/pod-account'
     version: 0.0.0
     dependencies:
@@ -20922,7 +20973,7 @@ packages:
     dev: false
 
   file:projects/pod-backup.tgz:
-    resolution: {integrity: sha512-dBeAnTqAnVqtSW51jV3itx5qXCHKKGrPwKGQ2uDdzcmoIyMvCF9Wvlsxaoo57svC6QHLbywpD2a3WK5D7npp8w==, tarball: file:projects/pod-backup.tgz}
+    resolution: {integrity: sha512-9cFf3HVgqNaOIXVWmLnR2grJ5I2nbx5QO6v+R3EKBfWWdX8/Jof6RHaHHnevqH7vbQSosDcXnV5BmO0kH10l9A==, tarball: file:projects/pod-backup.tgz}
     name: '@rush-temp/pod-backup'
     version: 0.0.0
     dependencies:
@@ -20947,20 +20998,13 @@ packages:
       ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
       typescript: 5.3.3
     transitivePeerDependencies:
-      - '@aws-sdk/credential-providers'
       - '@babel/core'
       - '@jest/types'
-      - '@mongodb-js/zstd'
       - '@swc/core'
       - '@swc/wasm'
       - babel-jest
       - babel-plugin-macros
-      - gcp-metadata
-      - kerberos
-      - mongodb-client-encryption
       - node-notifier
-      - snappy
-      - socks
       - supports-color
     dev: false
 

packages/core/src/client.ts

@@ -82,7 +82,8 @@ export enum ClientConnectEvent {
 
   // Client could cause back a few more states.
   Upgraded, // In case client code receive a full new model and need to be rebuild.
-  Refresh // In case we detect query refresh is required
+  Refresh, // In case we detect query refresh is required
+  Maintenance // In case workspace are in maintenance mode
 }
 /**
  * @public

plugins/client-resources/src/connection.ts

@@ -75,7 +75,7 @@ class Connection implements ClientConnection {
   private websocket: ClientSocket | Promise<ClientSocket> | null = null
   private readonly requests = new Map<ReqId, RequestPromise>()
   private lastId = 0
-  private readonly interval: number
+  private interval: number | undefined
   private sessionId: string | undefined
   private closed = false
 
@@ -86,21 +86,36 @@ class Connection implements ClientConnection {
   constructor (
     private readonly url: string,
     private readonly handler: TxHandler,
+    readonly workspace: string,
+    readonly email: string,
     private readonly onUpgrade?: () => void,
     private readonly onUnauthorized?: () => void,
-    readonly onConnect?: (event: ClientConnectEvent) => Promise<void>
-  ) {
+    readonly onConnect?: (event: ClientConnectEvent, data?: any) => Promise<void>
+  ) {}
+
+  private schedulePing (): void {
+    clearInterval(this.interval)
     this.interval = setInterval(() => {
+      if (this.upgrading) {
+        // no need to check while upgrade waiting
+        return
+      }
       if (this.pingResponse !== 0 && Date.now() - this.pingResponse > hangTimeout) {
         // No ping response from server.
         const s = this.websocket
 
         if (!(s instanceof Promise)) {
-          console.log('no ping response from server. Closing socket.', s, (s as any)?.readyState)
+          console.log(
+            'no ping response from server. Closing socket.',
+            this.workspace,
+            this.email,
+            s,
+            (s as any)?.readyState
+          )
           // Trying to close connection and re-establish it.
           s?.close(1000)
         } else {
-          console.log('no ping response from server. Closing socket.', s)
+          console.log('no ping response from server. Closing socket.', this.workspace, this.email, s)
           void s.then((s) => {
             s.close(1000)
           })
@@ -110,9 +125,7 @@ class Connection implements ClientConnection {
 
       if (!this.closed) {
         // eslint-disable-next-line @typescript-eslint/no-floating-promises
-        void this.sendRequest({ method: 'ping', params: [] }).then(() => {
-          this.pingResponse = Date.now()
-        })
+        void this.sendRequest({ method: 'ping', params: [] })
       } else {
         clearInterval(this.interval)
       }
@@ -160,13 +173,13 @@ class Connection implements ClientConnection {
         return await this.pending
       } catch (err: any) {
         if (this.closed) {
-          throw new Error('connection closed')
+          throw new Error('connection closed + ' + this.workspace + ' user: ' + this.email)
         }
         this.pending = undefined
         if (!this.upgrading) {
-          console.log('connection: failed to connect', this.lastId)
+          console.log('connection: failed to connect', `requests: ${this.lastId}`, this.workspace, this.email)
         } else {
-          console.log('connection: workspace during upgrade', this.lastId)
+          console.log('connection: workspace during upgrade', `requests: ${this.lastId}`, this.workspace, this.email)
         }
         if (err?.code === UNAUTHORIZED.code) {
           Analytics.handleError(err)
@@ -176,7 +189,7 @@ class Connection implements ClientConnection {
         await new Promise((resolve) => {
           setTimeout(() => {
             if (!this.upgrading) {
-              console.log(`delay ${this.delay} second`)
+              console.log(`delay ${this.delay} second`, this.workspace, this.email)
             }
             resolve(null)
             if (this.delay < 5) {
@@ -227,12 +240,24 @@ class Connection implements ClientConnection {
       }, dialTimeout)
 
       websocket.onmessage = (event: MessageEvent) => {
+        this.pingResponse = Date.now()
         const resp = readResponse<any>(event.data, binaryResponse)
-        if (resp.id === -1 && resp.result === 'upgrading') {
+        if (resp.id === -1 && resp.result.state === 'upgrading') {
+          void this.onConnect?.(ClientConnectEvent.Maintenance, resp.result.stats)
           this.upgrading = true
           return
         }
         if (resp.id === -1 && resp.result === 'hello') {
+          if (this.upgrading) {
+            // We need to call upgrade since connection is upgraded
+            this.onUpgrade?.()
+          }
+
+          console.log('connection established', this.workspace, this.email)
+
+          // Ok we connected, let's schedule ping
+          this.schedulePing()
+
           this.upgrading = false
           if ((resp as HelloResponse).alreadyConnected === true) {
             this.sessionId = generateId()
@@ -265,7 +290,7 @@ class Connection implements ClientConnection {
         if (resp.id !== undefined) {
           const promise = this.requests.get(resp.id)
           if (promise === undefined) {
-            throw new Error(`unknown response id: ${resp.id as string}`)
+            throw new Error(`unknown response id: ${resp.id as string} ${this.workspace} ${this.email}`)
           }
 
           if (resp.chunk !== undefined) {
@@ -303,7 +328,9 @@ class Connection implements ClientConnection {
               'error: ',
               resp.error,
               'result: ',
-              resp.result
+              resp.result,
+              this.workspace,
+              this.email
             )
             promise.reject(new PlatformError(resp.error))
           } else {
@@ -325,7 +352,7 @@ class Connection implements ClientConnection {
                 (tx as TxWorkspaceEvent).event === WorkspaceEvent.Upgrade) ||
               tx?._class === core.class.TxModelUpgrade
             ) {
-              console.log('Processing upgrade')
+              console.log('Processing upgrade', this.workspace, this.email)
               websocket.send(
                 serialize(
                   {
@@ -374,7 +401,7 @@ class Connection implements ClientConnection {
         websocket.send(serialize(helloRequest, false))
       }
       websocket.onerror = (event: any) => {
-        console.error('client websocket error:', socketId, event)
+        console.error('client websocket error:', socketId, event, this.workspace, this.email)
         void broadcastEvent(client.event.NetworkRequests, -1)
         reject(new Error(`websocket error:${socketId}`))
       }
@@ -526,11 +553,13 @@ class Connection implements ClientConnection {
 export async function connect (
   url: string,
   handler: TxHandler,
+  workspace: string,
+  user: string,
   onUpgrade?: () => void,
   onUnauthorized?: () => void,
-  onConnect?: (event: ClientConnectEvent) => void
+  onConnect?: (event: ClientConnectEvent, data?: any) => void
 ): Promise<ClientConnection> {
-  return new Connection(url, handler, onUpgrade, onUnauthorized, async (event) => {
-    onConnect?.(event)
+  return new Connection(url, handler, workspace, user, onUpgrade, onUnauthorized, async (event, data) => {
+    onConnect?.(event, data)
   })
 }

plugins/client-resources/src/index.ts

@@ -60,6 +60,18 @@ if (typeof localStorage !== 'undefined') {
   })
 }
 
+/**
+ * @public
+ */
+function decodeTokenPayload (token: string): any {
+  try {
+    return JSON.parse(atob(token.split('.')[1]))
+  } catch (err: any) {
+    console.error(err)
+    return {}
+  }
+}
+
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 export default async () => {
   return {
@@ -69,7 +81,7 @@ export default async () => {
         endpoint: string,
         onUpgrade?: () => void,
         onUnauthorized?: () => void,
-        onConnect?: (event: ClientConnectEvent) => void,
+        onConnect?: (event: ClientConnectEvent, data: any) => void,
         ctx?: MeasureContext
       ): Promise<AccountClient> => {
         const filterModel = getMetadata(clientPlugin.metadata.FilterModel) ?? false
@@ -95,8 +107,16 @@ export default async () => {
               }
               handler(...txes)
             }
-
-            return connect(url.href, upgradeHandler, onUpgrade, onUnauthorized, onConnect)
+            const tokenPayload: { workspace: string, email: string } = decodeTokenPayload(token)
+            return connect(
+              url.href,
+              upgradeHandler,
+              tokenPayload.workspace,
+              tokenPayload.email,
+              onUpgrade,
+              onUnauthorized,
+              onConnect
+            )
           },
           filterModel ? [...getPlugins(), ...(getMetadata(clientPlugin.metadata.ExtraPlugins) ?? [])] : undefined,
           createModelPersistence(getWSFromToken(token)),

plugins/client/src/index.ts

@@ -66,7 +66,7 @@ export type ClientFactory = (
   endpoint: string,
   onUpgrade?: () => void,
   onUnauthorized?: () => void,
-  onConnect?: (event: ClientConnectEvent) => void,
+  onConnect?: (event: ClientConnectEvent, data: any) => void,
   ctx?: MeasureContext
 ) => Promise<AccountClient>
 

plugins/workbench-resources/src/components/Workbench.svelte

@@ -180,7 +180,6 @@
   })
 
   const doSyncLoc = reduceCalls(async (loc: Location): Promise<void> => {
-    console.log('do sync', JSON.stringify(loc), $location.path)
     if (workspaceId !== $location.path[1]) {
       // Switch of workspace
       return
@@ -191,7 +190,6 @@
     await syncLoc(loc)
     await updateWindowTitle(loc)
     checkOnHide()
-    console.log('do sync-end', JSON.stringify(loc), $location.path)
   })
 
   onDestroy(

plugins/workbench-resources/src/components/WorkbenchApp.svelte

@@ -62,6 +62,11 @@
               {$workspaceCreating} %
             </div>
           {/if}
+          {#if $versionError}
+            <div class="ml-1">
+              {$versionError}
+            </div>
+          {/if}
         </Loading>
       {:then client}
         {#if $versionError}

plugins/workbench-resources/src/connect.ts

@@ -148,8 +148,16 @@ export async function connect (title: string): Promise<Client | undefined> {
           })
         },
         // We need to refresh all active live queries and clear old queries.
-        (event: ClientConnectEvent) => {
+        (event: ClientConnectEvent, data: any) => {
           console.log('WorkbenchClient: onConnect', event)
+          if (event === ClientConnectEvent.Maintenance) {
+            if (data !== undefined && data.total !== 0) {
+              versionError.set(`Maintenance ${Math.floor((100 / data.total) * (data.total - data.toProcess))}%`)
+            } else {
+              versionError.set('Maintenance...')
+            }
+            return
+          }
           try {
             if ((_clientSet && event === ClientConnectEvent.Connected) || event === ClientConnectEvent.Refresh) {
               void ctx.with('refresh client', {}, async () => {

pods/account/package.json

@@ -51,6 +51,7 @@
   },
   "dependencies": {
     "@hcengineering/account": "^0.6.0",
+    "@hcengineering/account-service": "^0.6.0",
     "@hcengineering/platform": "^0.6.9",
     "@hcengineering/auth-providers": "^0.6.0",
     "@hcengineering/core": "^0.6.28",

pods/account/src/__start.ts

@@ -13,9 +13,9 @@
 // limitations under the License.
 //
 
+import { serveAccount } from '@hcengineering/account-service'
 import { MeasureMetricsContext, newMetrics, type Tx } from '@hcengineering/core'
 import builder, { getModelVersion, migrateOperations } from '@hcengineering/model-all'
-import { serveAccount } from '.'
 
 const enabled = (process.env.MODEL_ENABLED ?? '*').split(',').map((it) => it.trim())
 const disabled = (process.env.MODEL_DISABLED ?? '').split(',').map((it) => it.trim())
@@ -24,4 +24,4 @@ const txes = JSON.parse(JSON.stringify(builder(enabled, disabled).getTxes())) as
 
 const metricsContext = new MeasureMetricsContext('account', {}, {}, newMetrics())
 
-serveAccount(metricsContext, getModelVersion(), txes, migrateOperations)
+serveAccount(metricsContext, getModelVersion(), txes, migrateOperations, '')

rush.json

@@ -821,6 +821,11 @@
       "projectFolder": "server/account",
       "shouldPublish": false
     },
+    {
+      "packageName": "@hcengineering/account-service",
+      "projectFolder": "server/account-service",
+      "shouldPublish": false
+    },
     {
       "packageName": "@hcengineering/collaborator",
       "projectFolder": "server/collaborator",

server/account-service/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+  extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: './tsconfig.json'
+  }
+}

server/account-service/.npmignore

@@ -0,0 +1,4 @@
+*
+!/lib/**
+!CHANGELOG.md
+/lib/**/__tests__/

server/account-service/build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+#  Copyright © 2020, 2021 Anticrm Platform Contributors.
+#  Copyright © 2021 Hardcore Engineering Inc.
+#  
+#  Licensed under the Eclipse Public License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License. You may
+#  obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+rushx bundle
+rushx docker:build
+rushx docker:push

server/account-service/config/rig.json

@@ -0,0 +1,5 @@
+{  
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+  "rigPackageName": "@hcengineering/platform-rig",
+  "rigProfile": "node"
+}

server/account-service/jest.config.js

@@ -0,0 +1,7 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
+  roots: ["./src"],
+  coverageReporters: ["text-summary", "html"]
+}

server/account-service/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@hcengineering/account-service",
+  "version": "0.6.0",
+  "main": "lib/index.js",
+  "svelte": "src/index.ts",
+  "types": "types/index.d.ts",
+  "author": "Anticrm Platform Contributors",
+  "template": "@hcengineering/node-package",
+  "license": "EPL-2.0",
+  "scripts": {
+    "start": "ts-node src/__start.ts",
+    "build": "compile",
+    "build:watch": "compile",
+    "format": "format src",
+    "test": "jest --passWithNoTests --silent --forceExit",
+    "_phase:build": "compile transpile src",
+    "_phase:test": "jest --passWithNoTests --silent --forceExit",
+    "_phase:format": "format src",
+    "_phase:validate": "compile validate"
+  },
+  "devDependencies": {
+    "cross-env": "~7.0.3",
+    "@hcengineering/platform-rig": "^0.6.0",
+    "@types/node": "~20.11.16",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint": "^8.54.0",
+    "esbuild": "^0.20.0",
+    "@types/koa-bodyparser": "^4.3.3",
+    "@types/koa-router": "^7.4.4",
+    "@types/koa": "2.14.0",
+    "@types/koa__cors": "^3.0.3",
+    "@typescript-eslint/parser": "^6.11.0",
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "prettier": "^3.1.0",
+    "ts-node": "^10.8.0",
+    "typescript": "^5.3.3",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.1",
+    "@types/jest": "^29.5.5"
+  },
+  "dependencies": {
+    "@hcengineering/account": "^0.6.0",
+    "@hcengineering/model": "^0.6.7",
+    "@hcengineering/platform": "^0.6.9",
+    "@hcengineering/auth-providers": "^0.6.0",
+    "@hcengineering/core": "^0.6.28",
+    "mongodb": "^6.3.0",
+    "koa": "^2.13.1",
+    "koa-router": "^12.0.1",
+    "koa-bodyparser": "^4.3.0",
+    "@koa/cors": "^3.1.0",
+    "@hcengineering/server-tool": "^0.6.0",
+    "@hcengineering/server-token": "^0.6.7",
+    "@hcengineering/analytics": "^0.6.0"
+  }
+}

server/account-service/src/index.ts

@@ -1,17 +1,5 @@
 //
-// Copyright © 2020, 2021 Anticrm Platform Contributors.
-// Copyright © 2021 Hardcore Engineering Inc.
-//
-// Licensed under the Eclipse Public License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License. You may
-// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//
-// See the License for the specific language governing permissions and
-// limitations under the License.
+// Copyright © 2023 Hardcore Engineering Inc.
 //
 
 import account, {
@@ -23,9 +11,10 @@ import account, {
 } from '@hcengineering/account'
 import accountEn from '@hcengineering/account/lang/en.json'
 import accountRu from '@hcengineering/account/lang/ru.json'
+import { Analytics } from '@hcengineering/analytics'
 import { registerProviders } from '@hcengineering/auth-providers'
 import { type Data, type MeasureContext, type Tx, type Version } from '@hcengineering/core'
-import { getModelVersion, type MigrateOperation } from '@hcengineering/model-all'
+import { type MigrateOperation } from '@hcengineering/model'
 import platform, { Severity, Status, addStringsLoader, setMetadata } from '@hcengineering/platform'
 import serverToken from '@hcengineering/server-token'
 import toolPlugin from '@hcengineering/server-tool'
@@ -44,9 +33,10 @@ export function serveAccount (
   version: Data<Version>,
   txes: Tx[],
   migrateOperations: [string, MigrateOperation][],
-  productId: string = ''
+  productId: string,
+  onClose?: () => void
 ): void {
-  const methods = getMethods(getModelVersion(), txes, migrateOperations)
+  const methods = getMethods(version, txes, migrateOperations)
   const ACCOUNT_PORT = parseInt(process.env.ACCOUNT_PORT ?? '3000')
   const dbUri = process.env.MONGO_URL
   if (dbUri === undefined) {
@@ -104,6 +94,8 @@ export function serveAccount (
   const app = new Koa()
   const router = new Router()
 
+  let worker: UpgradeWorker | undefined
+
   void client.then(async (p: MongoClient) => {
     const db = p.db(ACCOUNT_DB)
     registerProviders(measureCtx, app, router, db, productId, serverSecret, frontURL)
@@ -111,9 +103,11 @@ export function serveAccount (
     // We need to clean workspace with creating === true, since server is restarted.
     void cleanInProgressWorkspaces(db, productId)
 
-    const worker = new UpgradeWorker(db, p, version, txes, migrateOperations, productId)
+    worker = new UpgradeWorker(db, p, version, txes, migrateOperations, productId)
     await worker.upgradeAll(measureCtx, {
-      errorHandler: async (ws, err) => {},
+      errorHandler: async (ws, err) => {
+        Analytics.handleError(err)
+      },
       force: false,
       console: false,
       logs: 'upgrade-logs',
@@ -148,6 +142,8 @@ export function serveAccount (
     }
     const db = client.db(ACCOUNT_DB)
     const result = await method(measureCtx, db, productId, request, token)
+
+    worker?.updateResponseStatistics(result)
     ctx.body = result
   })
 
@@ -164,6 +160,7 @@ export function serveAccount (
   })
 
   const close = (): void => {
+    onClose?.()
     if (client instanceof Promise) {
       void client.then((c) => c.close())
     } else {
@@ -173,13 +170,13 @@ export function serveAccount (
   }
 
   process.on('uncaughtException', (e) => {
-    console.error(e)
+    void measureCtx.error('uncaughtException', { error: e })
   })
 
   process.on('unhandledRejection', (reason, promise) => {
     console.error('Unhandled Rejection at:', promise, 'reason:', reason)
+    void measureCtx.error('Unhandled Rejection at:', { reason, promise })
   })
-
   process.on('SIGINT', close)
   process.on('SIGTERM', close)
   process.on('exit', close)

server/account-service/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json",
+
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./lib",
+    "declarationDir": "./types",
+    "tsBuildInfoFile": ".build/build.tsbuildinfo"
+  }
+}

server/account/src/service.ts

@@ -43,8 +43,18 @@ export class UpgradeWorker {
   canceled = false
 
   st: number = Date.now()
-  workspaces: BaseWorkspaceInfo[] = []
+  total: number = 0
   toProcess: number = 0
+  eta: number = 0
+
+  updateResponseStatistics (response: any): void {
+    response.upgrade = {
+      toProcess: this.toProcess,
+      total: this.total,
+      elapsed: Date.now() - this.st,
+      eta: this.eta
+    }
+  }
 
   async close (): Promise<void> {
     this.canceled = true
@@ -67,10 +77,11 @@ export class UpgradeWorker {
 
     const logger = opt.console ? ctxModelLogger : new FileModelLogger(path.join(opt.logs, `${ws.workspace}.log`))
 
-    const avgTime = (Date.now() - this.st) / (this.workspaces.length - this.toProcess + 1)
+    const avgTime = (Date.now() - this.st) / (this.total - this.toProcess + 1)
+    this.eta = Math.floor(avgTime * this.toProcess)
     await ctx.info('----------------------------------------------------------\n---UPGRADING----', {
       pending: this.toProcess,
-      eta: Math.floor(avgTime * this.toProcess),
+      eta: this.eta,
       workspace: ws.workspace
     })
     this.toProcess--
@@ -132,6 +143,7 @@ export class UpgradeWorker {
     const withError: string[] = []
     this.toProcess = workspaces.length
     this.st = Date.now()
+    this.total = workspaces.length
 
     if (opt.parallel !== 0) {
       const parallel = opt.parallel

server/ws/src/server.ts

@@ -45,7 +45,14 @@ import {
   type Workspace
 } from './types'
 
-interface WorkspaceLoginInfo extends BaseWorkspaceInfo {}
+interface WorkspaceLoginInfo extends BaseWorkspaceInfo {
+  upgrade?: {
+    toProcess: number
+    total: number
+    elapsed: number
+    eta: number
+  }
+}
 
 function timeoutPromise (time: number): Promise<void> {
   return new Promise((resolve) => {
@@ -180,7 +187,7 @@ class TSessionManager implements SessionManager {
     return this.sessionFactory(token, pipeline, this.broadcast.bind(this))
   }
 
-  async getWorkspaceInfo (accounts: string, token: string): Promise<BaseWorkspaceInfo> {
+  async getWorkspaceInfo (accounts: string, token: string): Promise<WorkspaceLoginInfo> {
     const userInfo = await (
       await fetch(accounts, {
         method: 'POST',
@@ -195,7 +202,7 @@ class TSessionManager implements SessionManager {
       })
     ).json()
 
-    return userInfo.result as WorkspaceLoginInfo
+    return { ...userInfo.result, upgrade: userInfo.upgrade }
   }
 
   async addSession (
@@ -208,7 +215,9 @@ class TSessionManager implements SessionManager {
     sessionId: string | undefined,
     accountsUrl: string
   ): Promise<
-    { session: Session, context: MeasureContext, workspaceName: string } | { upgrade: true } | { error: any }
+    | { session: Session, context: MeasureContext, workspaceName: string }
+    | { upgrade: true, upgradeInfo?: WorkspaceLoginInfo['upgrade'] }
+    | { error: any }
     > {
     return await baseCtx.with('📲 add-session', {}, async (ctx) => {
       const wsString = toWorkspaceString(token.workspace, '@')
@@ -239,7 +248,7 @@ class TSessionManager implements SessionManager {
           workspaceVersion: versionToString(workspaceInfo.version)
         })
         // Version mismatch, return upgrading.
-        return { upgrade: true }
+        return { upgrade: true, upgradeInfo: workspaceInfo.upgrade }
       }
 
       let workspace = this.workspaces.get(wsString)
@@ -319,7 +328,7 @@ class TSessionManager implements SessionManager {
     })
   }
 
-  private wsFromToken (token: Token): BaseWorkspaceInfo {
+  private wsFromToken (token: Token): WorkspaceLoginInfo {
     return {
       workspace: token.workspace.name,
       workspaceUrl: token.workspace.name,

server/ws/src/server_http.ts

@@ -223,11 +223,12 @@ export function startHttpServer (
       if ('error' in session) {
         void ctx.error('error', { error: session.error?.message, stack: session.error?.stack })
       }
-      await cs.send(ctx, { id: -1, result: 'upgrading' }, false, false)
+      await cs.send(ctx, { id: -1, result: { state: 'upgrading', stats: (session as any).upgradeInfo } }, false, false)
+
       // Wait 1 second before closing the connection
       setTimeout(() => {
         cs.close()
-      }, 1000)
+      }, 10000)
       return
     }
     // eslint-disable-next-line @typescript-eslint/no-misused-promises