diff --git a/server/client/src/account.ts b/server/client/src/account.ts index c2bac8440e..b73dfe448a 100644 --- a/server/client/src/account.ts +++ b/server/client/src/account.ts @@ -102,30 +102,47 @@ export async function getTransactorEndpoint ( } } -export function withRetryUntilTimeout

( +export function withRetry

( f: (...params: P) => Promise, - timeoutMs: number = 5000 + shouldFail: (err: any, attempt: number) => boolean, + intervalMs: number = 1000 ): (...params: P) => Promise { return async function (...params: P): Promise { - const timeout = Date.now() + timeoutMs + let attempt = 0 while (true) { try { return await f(...params) } catch (err: any) { - if (timeout < Date.now()) { - // Timeout happened - throw err - } - if (err?.cause?.code === 'ECONNRESET' || err?.cause?.code === 'ECONNREFUSED') { - await new Promise((resolve) => setTimeout(resolve, 1000)) - } else { + if (shouldFail(err, attempt)) { throw err } + + attempt++ + await new Promise((resolve) => setTimeout(resolve, intervalMs)) } } } } +export function withRetryConnUntilTimeout

( + f: (...params: P) => Promise, + timeoutMs: number = 5000 +): (...params: P) => Promise { + const timeout = Date.now() + timeoutMs + const shouldFail = (err: any): boolean => + (err?.cause?.code !== 'ECONNRESET' && err?.cause?.code !== 'ECONNREFUSED') || timeout < Date.now() + + return withRetry(f, shouldFail) +} + +export function withRetryConnUntilSuccess

( + f: (...params: P) => Promise +): (...params: P) => Promise { + const shouldFail = (err: any): boolean => err?.cause?.code !== 'ECONNRESET' && err?.cause?.code !== 'ECONNREFUSED' + + return withRetry(f, shouldFail) +} + export async function getPendingWorkspace ( token: string, region: string, diff --git a/server/workspace-service/src/service.ts b/server/workspace-service/src/service.ts index 3486edd28d..9a0c58f757 100644 --- a/server/workspace-service/src/service.ts +++ b/server/workspace-service/src/service.ts @@ -28,7 +28,8 @@ import { getPendingWorkspace, updateWorkspaceInfo, workerHandshake, - withRetryUntilTimeout + withRetryConnUntilTimeout, + withRetryConnUntilSuccess } from '@hcengineering/server-client' import { generateToken } from '@hcengineering/server-token' import { FileModelLogger } from '@hcengineering/server-tool' @@ -84,7 +85,12 @@ export class WorkspaceWorker { } this.wakeup = this.defaultWakeup const token = generateToken(systemAccountEmail, { name: '-' }, { service: 'workspace' }) - await workerHandshake(token, this.region, this.version, this.operation) + + ctx.info('Sending a handshake to the account service...') + + await withRetryConnUntilSuccess(workerHandshake)(token, this.region, this.version, this.operation) + + ctx.info('Successfully connected to the account service') while (true) { await this.waitForAvailableThread() @@ -156,7 +162,7 @@ export class WorkspaceWorker { progress: number, message?: string ): Promise => { - return withRetryUntilTimeout( + return withRetryConnUntilTimeout( () => updateWorkspaceInfo(token, ws.workspace, event, version, progress, message), 5000 )() @@ -239,7 +245,7 @@ export class WorkspaceWorker { progress: number, message?: string ): Promise => { - return withRetryUntilTimeout( + return withRetryConnUntilTimeout( () => updateWorkspaceInfo(token, ws.workspace, event, version, progress, message), 5000 )()