diff --git a/server/client/src/account.ts b/server/client/src/account.ts
index c2bac8440e..b73dfe448a 100644
--- a/server/client/src/account.ts
+++ b/server/client/src/account.ts
@@ -102,30 +102,47 @@ export async function getTransactorEndpoint (
}
}
-export function withRetryUntilTimeout
(
+export function withRetry
(
f: (...params: P) => Promise,
- timeoutMs: number = 5000
+ shouldFail: (err: any, attempt: number) => boolean,
+ intervalMs: number = 1000
): (...params: P) => Promise {
return async function (...params: P): Promise {
- const timeout = Date.now() + timeoutMs
+ let attempt = 0
while (true) {
try {
return await f(...params)
} catch (err: any) {
- if (timeout < Date.now()) {
- // Timeout happened
- throw err
- }
- if (err?.cause?.code === 'ECONNRESET' || err?.cause?.code === 'ECONNREFUSED') {
- await new Promise((resolve) => setTimeout(resolve, 1000))
- } else {
+ if (shouldFail(err, attempt)) {
throw err
}
+
+ attempt++
+ await new Promise((resolve) => setTimeout(resolve, intervalMs))
}
}
}
}
+export function withRetryConnUntilTimeout (
+ f: (...params: P) => Promise,
+ timeoutMs: number = 5000
+): (...params: P) => Promise {
+ const timeout = Date.now() + timeoutMs
+ const shouldFail = (err: any): boolean =>
+ (err?.cause?.code !== 'ECONNRESET' && err?.cause?.code !== 'ECONNREFUSED') || timeout < Date.now()
+
+ return withRetry(f, shouldFail)
+}
+
+export function withRetryConnUntilSuccess (
+ f: (...params: P) => Promise
+): (...params: P) => Promise {
+ const shouldFail = (err: any): boolean => err?.cause?.code !== 'ECONNRESET' && err?.cause?.code !== 'ECONNREFUSED'
+
+ return withRetry(f, shouldFail)
+}
+
export async function getPendingWorkspace (
token: string,
region: string,
diff --git a/server/workspace-service/src/service.ts b/server/workspace-service/src/service.ts
index 3486edd28d..9a0c58f757 100644
--- a/server/workspace-service/src/service.ts
+++ b/server/workspace-service/src/service.ts
@@ -28,7 +28,8 @@ import {
getPendingWorkspace,
updateWorkspaceInfo,
workerHandshake,
- withRetryUntilTimeout
+ withRetryConnUntilTimeout,
+ withRetryConnUntilSuccess
} from '@hcengineering/server-client'
import { generateToken } from '@hcengineering/server-token'
import { FileModelLogger } from '@hcengineering/server-tool'
@@ -84,7 +85,12 @@ export class WorkspaceWorker {
}
this.wakeup = this.defaultWakeup
const token = generateToken(systemAccountEmail, { name: '-' }, { service: 'workspace' })
- await workerHandshake(token, this.region, this.version, this.operation)
+
+ ctx.info('Sending a handshake to the account service...')
+
+ await withRetryConnUntilSuccess(workerHandshake)(token, this.region, this.version, this.operation)
+
+ ctx.info('Successfully connected to the account service')
while (true) {
await this.waitForAvailableThread()
@@ -156,7 +162,7 @@ export class WorkspaceWorker {
progress: number,
message?: string
): Promise => {
- return withRetryUntilTimeout(
+ return withRetryConnUntilTimeout(
() => updateWorkspaceInfo(token, ws.workspace, event, version, progress, message),
5000
)()
@@ -239,7 +245,7 @@ export class WorkspaceWorker {
progress: number,
message?: string
): Promise => {
- return withRetryUntilTimeout(
+ return withRetryConnUntilTimeout(
() => updateWorkspaceInfo(token, ws.workspace, event, version, progress, message),
5000
)()