UBERF-8333: Retry WS handshake (#6786)

Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com>

server/client/src/account.ts

@@ -102,30 +102,47 @@ export async function getTransactorEndpoint (
   }
 }
 
-export function withRetryUntilTimeout<P extends any[], T> (
+export function withRetry<P extends any[], T> (
   f: (...params: P) => Promise<T>,
-  timeoutMs: number = 5000
+  shouldFail: (err: any, attempt: number) => boolean,
+  intervalMs: number = 1000
 ): (...params: P) => Promise<T> {
   return async function (...params: P): Promise<T> {
-    const timeout = Date.now() + timeoutMs
+    let attempt = 0
     while (true) {
       try {
         return await f(...params)
       } catch (err: any) {
-        if (timeout < Date.now()) {
+        if (shouldFail(err, attempt)) {
-          // Timeout happened
-          throw err
-        }
-        if (err?.cause?.code === 'ECONNRESET' || err?.cause?.code === 'ECONNREFUSED') {
-          await new Promise<void>((resolve) => setTimeout(resolve, 1000))
-        } else {
           throw err
         }
+
+        attempt++
+        await new Promise<void>((resolve) => setTimeout(resolve, intervalMs))
       }
     }
   }
 }
 
+export function withRetryConnUntilTimeout<P extends any[], T> (
+  f: (...params: P) => Promise<T>,
+  timeoutMs: number = 5000
+): (...params: P) => Promise<T> {
+  const timeout = Date.now() + timeoutMs
+  const shouldFail = (err: any): boolean =>
+    (err?.cause?.code !== 'ECONNRESET' && err?.cause?.code !== 'ECONNREFUSED') || timeout < Date.now()
+
+  return withRetry(f, shouldFail)
+}
+
+export function withRetryConnUntilSuccess<P extends any[], T> (
+  f: (...params: P) => Promise<T>
+): (...params: P) => Promise<T> {
+  const shouldFail = (err: any): boolean => err?.cause?.code !== 'ECONNRESET' && err?.cause?.code !== 'ECONNREFUSED'
+
+  return withRetry(f, shouldFail)
+}
+
 export async function getPendingWorkspace (
   token: string,
   region: string,

server/workspace-service/src/service.ts

@@ -28,7 +28,8 @@ import {
   getPendingWorkspace,
   updateWorkspaceInfo,
   workerHandshake,
-  withRetryUntilTimeout
+  withRetryConnUntilTimeout,
+  withRetryConnUntilSuccess
 } from '@hcengineering/server-client'
 import { generateToken } from '@hcengineering/server-token'
 import { FileModelLogger } from '@hcengineering/server-tool'
@@ -84,7 +85,12 @@ export class WorkspaceWorker {
     }
     this.wakeup = this.defaultWakeup
     const token = generateToken(systemAccountEmail, { name: '-' }, { service: 'workspace' })
-    await workerHandshake(token, this.region, this.version, this.operation)
+
+    ctx.info('Sending a handshake to the account service...')
+
+    await withRetryConnUntilSuccess(workerHandshake)(token, this.region, this.version, this.operation)
+
+    ctx.info('Successfully connected to the account service')
 
     while (true) {
       await this.waitForAvailableThread()
@@ -156,7 +162,7 @@ export class WorkspaceWorker {
         progress: number,
         message?: string
       ): Promise<void> => {
-        return withRetryUntilTimeout(
+        return withRetryConnUntilTimeout(
           () => updateWorkspaceInfo(token, ws.workspace, event, version, progress, message),
           5000
         )()
@@ -239,7 +245,7 @@ export class WorkspaceWorker {
         progress: number,
         message?: string
       ): Promise<void> => {
-        return withRetryUntilTimeout(
+        return withRetryConnUntilTimeout(
           () => updateWorkspaceInfo(token, ws.workspace, event, version, progress, message),
           5000
         )()