UBERF-11111: Add retry package

Signed-off-by: Artem Savchenko <armisav@gmail.com>

utils/retry/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+  extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: './tsconfig.json'
+  }
+}

utils/retry/.npmignore

@@ -0,0 +1,4 @@
+*
+!/lib/**
+!CHANGELOG.md
+/lib/**/__tests__/

utils/retry/config/rig.json

@@ -0,0 +1,5 @@
+{  
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+  "rigPackageName": "@hcengineering/platform-rig",
+  "rigProfile": "node"
+}

utils/retry/jest.config.js

@@ -0,0 +1,7 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
+  roots: ["./src"],
+  coverageReporters: ["text-summary", "html"]
+}

utils/retry/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@hcengineering/retry",
+  "version": "0.6.0",
+  "main": "lib/index.js",
+  "svelte": "src/index.ts",
+  "types": "types/index.d.ts",
+  "author": "Copyright © Hardcore Engineering Inc.",
+  "template": "@hcengineering/node-package",
+  "license": "EPL-2.0",
+  "scripts": {
+    "build": "compile",
+    "build:watch": "compile",
+    "test": "jest --passWithNoTests --silent --forceExit",
+    "format": "format src",
+    "_phase:build": "compile transpile src",
+    "_phase:test": "jest --passWithNoTests --silent --forceExit",
+    "_phase:format": "format src",
+    "_phase:validate": "compile validate"
+  },
+  "devDependencies": {
+    "@hcengineering/platform-rig": "^0.6.0",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint": "^8.54.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "prettier": "^3.1.0",
+    "typescript": "^5.3.3",
+    "@types/node": "~20.11.16",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.1",
+    "@types/jest": "^29.5.5"
+  },
+  "dependencies": {
+  }
+}

utils/retry/readme.md

@@ -0,0 +1,193 @@
+# Retry Utility
+
+A comprehensive TypeScript utility for handling transient failures with exponential backoff, jitter, and customizable retry conditions.
+
+## Features
+
+- ✅ **Exponential backoff** with configurable parameters
+- ✅ **Jitter support** to prevent thundering herd problems
+- ✅ **Customizable retry conditions** to control which errors should be retried
+- ✅ **TypeScript decorators** for clean, declarative retry logic
+- ✅ **Function wrappers** for retrofitting existing code with retry capabilities
+- ✅ **Comprehensive logging** of retry attempts and failures
+
+## Usage
+
+### Basic Usage
+
+Wrap any async operation with the `withRetry` function:
+
+```typescript
+import { withRetry } from '@hcengineering/retry'
+
+async function fetchData() {
+  const data = await withRetry(
+    async () => {
+      // Your async operation that might fail transiently
+      return await api.getData()
+    },
+    { maxRetries: 3 }
+  )
+  return data
+}
+```
+
+### Using Decorators
+
+For class methods, you can use the `@Retryable` decorator for clean, declarative retry logic:
+
+```typescript
+import { Retryable } from '@hcengineering/retry'
+
+class UserService {
+  @Retryable({ maxRetries: 3, initialDelayMs: 500 })
+  async getUserProfile(userId: string): Promise<UserProfile> {
+    // This method will automatically retry on failure
+    return await this.api.fetchUserProfile(userId)
+  }
+}
+```
+
+### Custom Retry Conditions
+
+You can specify which errors should trigger retries:
+
+```typescript
+import { withRetry, retryNetworkErrors } from '@platform/utils/retry'
+
+async function fetchData() {
+  return await withRetry(
+    async () => await api.getData(),
+    {
+      // Only retry network-related errors
+      isRetryable: retryNetworkErrors,
+      maxRetries: 5
+    }
+  )
+}
+```
+
+Create your own custom retry condition:
+
+```typescript
+import { type IsRetryable } from '@platform/utils/retry'
+
+// Custom retry condition
+const retryDatabaseErrors: IsRetryable = (error: unknown): boolean => {
+  if (error instanceof DatabaseError) {
+    // Only retry specific database errors
+    return error.code === 'CONNECTION_LOST' || 
+           error.code === 'DEADLOCK' || 
+           error.code === 'TIMEOUT'
+  }
+  return false
+}
+
+// Use it
+await withRetry(
+  async () => await db.query('SELECT * FROM users'),
+  { isRetryable: retryDatabaseErrors }
+)
+```
+
+## API Reference
+
+### `withRetry<T>(operation, options?, operationName?): Promise<T>`
+
+Executes an async operation with retry logic.
+
+- `operation`: `() => Promise<T>` - The async operation to execute
+- `options`: `Partial<RetryOptions>` - Retry configuration (optional)
+- `operationName`: `string` - Name for logging (optional)
+- Returns: `Promise<T>` - The result of the operation
+
+### `createRetryableFunction<T>(fn, options?, operationName?): T`
+
+Creates a retryable function from an existing function.
+
+- `fn`: `T extends (...args: any[]) => Promise<any>` - The function to make retryable
+- `options`: `Partial<RetryOptions>` - Retry configuration (optional)
+- `operationName`: `string` - Name for logging (optional)
+- Returns: `T` - A wrapped function with retry logic
+
+### `@Retryable(options?)`
+
+Method decorator for adding retry functionality to class methods.
+
+- `options`: `Partial<RetryOptions>` - Retry configuration (optional)
+
+### RetryOptions
+
+Configuration options for the retry mechanism:
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `initialDelayMs` | `number` | `1000` | Initial delay between retries in milliseconds |
+| `maxDelayMs` | `number` | `30000` | Maximum delay between retries in milliseconds |
+| `maxRetries` | `number` | `5` | Maximum number of retry attempts |
+| `backoffFactor` | `number` | `1.5` | Backoff factor for exponential delay increase |
+| `jitter` | `number` | `0.2` | Jitter factor (0-1) to add randomness to delay times |
+| `isRetryable` | `IsRetryable` | `retryAllErrors` | Function to determine if an error is retriable |
+| `logger` | `Logger` | `defaultLogger` | Logger to use |
+
+### Retry Condition Functions
+
+| Function | Description |
+|----------|-------------|
+| `retryAllErrors` | Retry on any error (default) |
+| `retryNetworkErrors` | Retry only on network-related errors |
+
+## Examples
+
+### Basic Retry with Custom Options
+
+```typescript
+import { withRetry } from '@platform/utils/retry'
+
+async function fetchDataWithRetry() {
+  return await withRetry(
+    async () => {
+      const response = await fetch('https://api.example.com/data')
+      if (!response.ok) {
+        throw new Error(`HTTP error: ${response.status}`)
+      }
+      return await response.json()
+    },
+    {
+      initialDelayMs: 300,   // Start with 300ms delay
+      maxDelayMs: 10000,     // Max delay of 10 seconds
+      maxRetries: 4,         // Try up to 4 times (1 initial + 3 retries)
+      backoffFactor: 2,      // Double the delay each time
+      jitter: 0.25           // Add 25% randomness to delay
+    },
+    'fetchApiData'           // Name for logging
+  )
+}
+```
+
+### Class with Multiple Retryable Methods
+
+```typescript
+import { Retryable, retryNetworkErrors } from '@platform/utils/retry'
+
+class DataService {
+  @Retryable({
+    maxRetries: 3,
+    initialDelayMs: 200
+  })
+  async fetchUsers(): Promise<User[]> {
+    // Will retry up to 3 times with initial 200ms delay
+    return await this.api.getUsers()
+  }
+
+  @Retryable({
+    maxRetries: 5,
+    initialDelayMs: 1000,
+    isRetryable: retryNetworkErrors
+  })
+  async uploadFile(file: File): Promise<string> {
+    // Will retry up to 5 times, but only for network errors
+    return await this.api.uploadFile(file)
+  }
+}
+```

utils/retry/src/__test__/decorator.test.ts

@@ -0,0 +1,316 @@
+//
+// Copyright © 2025 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { Retryable } from '../decorator'
+import { type RetryOptions } from '../retry'
+
+// Instead of mocking withRetry, we'll mock setTimeout to avoid waiting in tests
+jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {
+  fn()
+  return 1 as any
+})
+
+describe('Retryable decorator', () => {
+  const mockLogger = {
+    warn: jest.fn(),
+    error: jest.fn(),
+    info: jest.fn()
+  }
+
+  const mockOptions: Partial<RetryOptions> = {
+    initialDelayMs: 10,
+    maxDelayMs: 100,
+    maxRetries: 3,
+    backoffFactor: 2,
+    logger: mockLogger
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+  })
+
+  it('should retry failed operations', async () => {
+    // Create a test class with decorated method that fails initially then succeeds
+    class TestService {
+      callCount = 0
+
+      @Retryable(mockOptions)
+      async testMethod (param1: string, param2: number): Promise<string> {
+        this.callCount++
+        if (this.callCount === 1) {
+          throw new Error('First attempt failed')
+        }
+        return `${param1}-${param2}`
+      }
+    }
+
+    const service = new TestService()
+    const result = await service.testMethod('test', 123)
+
+    // Check results
+    expect(result).toBe('test-123')
+    expect(service.callCount).toBe(2) // Called once, failed, then succeeded on retry
+
+    // Check logs
+    expect(mockLogger.warn).toHaveBeenCalledTimes(1)
+    expect(mockLogger.warn).toHaveBeenCalledWith(
+      expect.stringContaining('testMethod failed'),
+      expect.objectContaining({
+        error: 'First attempt failed',
+        attempt: 1
+      })
+    )
+    expect(mockLogger.error).not.toHaveBeenCalled()
+  })
+
+  it('should work with default options', async () => {
+    class TestService {
+      callCount = 0
+
+      @Retryable()
+      async testMethod (): Promise<string> {
+        this.callCount++
+        if (this.callCount === 1) {
+          throw new Error('First attempt failed')
+        }
+        return 'success'
+      }
+    }
+
+    const service = new TestService()
+    const result = await service.testMethod()
+
+    expect(result).toBe('success')
+    expect(service.callCount).toBe(2) // Should have retried once
+  })
+
+  it('should preserve class instance context (this)', async () => {
+    class TestService {
+      private counter = 0
+
+      @Retryable(mockOptions)
+      async incrementAndGet (): Promise<number> {
+        if (this.counter === 0) {
+          this.counter++
+          throw new Error('First attempt failed')
+        }
+        this.counter++
+        return this.counter
+      }
+
+      getCounter (): number {
+        return this.counter
+      }
+    }
+
+    const service = new TestService()
+    const result = await service.incrementAndGet()
+
+    // Check that the class context was preserved across retries
+    expect(result).toBe(2)
+    expect(service.getCounter()).toBe(2) // Incremented once per attempt
+  })
+
+  it('should throw after max retries are exhausted', async () => {
+    class TestService {
+      @Retryable({
+        ...mockOptions,
+        maxRetries: 2 // Only try twice total (initial + 1 retry)
+      })
+      async alwaysFailingMethod (): Promise<string> {
+        throw new Error('Persistent failure')
+      }
+    }
+
+    const service = new TestService()
+    await expect(service.alwaysFailingMethod()).rejects.toThrow('Persistent failure')
+
+    // Should have tried twice in total (initial + 1 retry)
+    expect(mockLogger.warn).toHaveBeenCalledTimes(1)
+    expect(mockLogger.error).toHaveBeenCalledTimes(1)
+    expect(mockLogger.error).toHaveBeenCalledWith(
+      expect.stringContaining('alwaysFailingMethod failed after 2 attempts'),
+      expect.any(Object)
+    )
+  })
+
+  it('should handle async methods correctly', async () => {
+    let resolutionCount = 0
+
+    // Create a class with an async method that fails then resolves
+    class TestService {
+      @Retryable(mockOptions)
+      async delayedMethod (): Promise<string> {
+        return await new Promise<string>((resolve, reject) => {
+          resolutionCount++
+          if (resolutionCount === 1) {
+            reject(new Error('Delayed error'))
+          } else {
+            resolve('delayed success')
+          }
+        })
+      }
+    }
+
+    const service = new TestService()
+    const result = await service.delayedMethod()
+
+    expect(result).toBe('delayed success')
+    expect(resolutionCount).toBe(2)
+    expect(mockLogger.warn).toHaveBeenCalledTimes(1)
+  })
+
+  it('should retry according to the specified retry count', async () => {
+    let callCount = 0
+
+    class TestService {
+      @Retryable({
+        maxRetries: 5, // Should try up to 5 times total
+        initialDelayMs: 10,
+        logger: mockLogger
+      })
+      async unstableMethod (): Promise<string> {
+        callCount++
+        if (callCount < 4) {
+          // Succeed on the 4th attempt
+          throw new Error(`Failure #${callCount}`)
+        }
+        return 'success after retries'
+      }
+    }
+
+    const service = new TestService()
+    const result = await service.unstableMethod()
+
+    expect(result).toBe('success after retries')
+    expect(callCount).toBe(4) // Initial attempt + 3 retries = 4 total calls
+    expect(mockLogger.warn).toHaveBeenCalledTimes(3) // Should have logged 3 warnings
+  })
+
+  it('should respect different delay settings', async () => {
+    // Override the setTimeout mock to capture delay values
+    const delayValues: number[] = []
+    jest.spyOn(global, 'setTimeout').mockImplementation((fn: any, delay: number | undefined) => {
+      delayValues.push(delay ?? 0)
+      fn()
+      return 1 as any
+    })
+
+    let callCount = 0
+
+    class TestService {
+      @Retryable({
+        maxRetries: 4,
+        initialDelayMs: 100,
+        maxDelayMs: 500,
+        backoffFactor: 2,
+        jitter: 0 // Disable jitter for predictable tests
+      })
+      async delayingMethod (): Promise<string> {
+        callCount++
+        if (callCount < 4) {
+          throw new Error(`Attempt ${callCount} failed`)
+        }
+        return 'success'
+      }
+    }
+
+    const service = new TestService()
+    await service.delayingMethod()
+
+    // Should have recorded 3 delays: initial, 2x initial, 4x initial (capped at maxDelayMs)
+    expect(delayValues).toHaveLength(3)
+    expect(delayValues[0]).toBe(100) // initial delay
+    expect(delayValues[1]).toBe(200) // 2x initial
+    expect(delayValues[2]).toBe(400) // 4x initial
+  })
+
+  it('should handle methods returning non-promises', async () => {
+    let callCount = 0
+
+    class TestService {
+      @Retryable(mockOptions)
+      nonAsyncMethod (input: string): string {
+        callCount++
+        if (callCount === 1) {
+          throw new Error('Sync error')
+        }
+        return `processed-${input}`
+      }
+    }
+
+    const service = new TestService()
+    // Even though the original method is not async, the decorated method returns a Promise
+    // eslint-disable-next-line @typescript-eslint/await-thenable
+    const result = await service.nonAsyncMethod('test')
+
+    expect(result).toBe('processed-test')
+    expect(callCount).toBe(2) // Should have retried once
+    expect(mockLogger.warn).toHaveBeenCalledTimes(1)
+  })
+
+  it('should handle static methods', async () => {
+    let callCount = 0
+
+    // eslint-disable-next-line @typescript-eslint/no-extraneous-class
+    class TestService {
+      @Retryable(mockOptions)
+      static async staticMethod (input: string): Promise<string> {
+        callCount++
+        if (callCount === 1) {
+          throw new Error('Static method error')
+        }
+        return `static-${input}`
+      }
+    }
+
+    const result = await TestService.staticMethod('test')
+
+    expect(result).toBe('static-test')
+    expect(callCount).toBe(2) // Should have retried once
+  })
+
+  it('should respect isRetryable option', async () => {
+    class TestService {
+      callCount = 0
+
+      @Retryable({
+        ...mockOptions,
+        isRetryable: (err) => {
+          // Only retry errors with "retry" in the message
+          return err instanceof Error && err.message.includes('Please retry')
+        }
+      })
+      async conditionalRetryMethod (): Promise<string> {
+        this.callCount++
+        if (this.callCount === 1) {
+          throw new Error('Please retry this') // should be retried
+        }
+        if (this.callCount === 2) {
+          throw new Error('Do not retry this') // should not be retried
+        }
+        return 'success'
+      }
+    }
+
+    const service = new TestService()
+    // Should fail with the second error since it won't be retried
+    await expect(service.conditionalRetryMethod()).rejects.toThrow('Do not retry this')
+
+    expect(service.callCount).toBe(2) // Should have called twice (original + 1 retry)
+    expect(mockLogger.warn).toHaveBeenCalledTimes(1) // Only first error logged as warning
+    expect(mockLogger.error).toHaveBeenCalledTimes(1) // Second error logged as error
+  })
+})

utils/retry/src/__test__/retry.test.ts

@@ -0,0 +1,478 @@
+//
+// Copyright © 2025 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { withRetry, createRetryableFunction, type RetryOptions } from '../retry'
+import { type IsRetryable, retryAllErrors } from '../retryable'
+
+// Mock the sleep function to speed up tests
+jest.mock('../retry', () => {
+  const originalModule = jest.requireActual('../retry')
+  return {
+    ...originalModule,
+    // Override the internal sleep function to resolve immediately
+    sleep: jest.fn().mockImplementation(() => Promise.resolve())
+  }
+})
+
+describe('withRetry', () => {
+  // Create a mock logger to capture logs
+  const mockLogger = {
+    warn: jest.fn(),
+    error: jest.fn(),
+    info: jest.fn()
+  }
+
+  const mockOptions: RetryOptions = {
+    initialDelayMs: 10,
+    maxDelayMs: 100,
+    maxRetries: 3,
+    backoffFactor: 2,
+    jitter: 0,
+    logger: mockLogger,
+    isRetryable: retryAllErrors
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+  })
+
+  it('should return the result when operation succeeds on first try', async () => {
+    const mockOperation = jest.fn().mockResolvedValue('success')
+
+    const result = await withRetry(mockOperation, mockOptions)
+
+    expect(result).toBe('success')
+    expect(mockOperation).toHaveBeenCalledTimes(1)
+    expect(mockLogger.warn).not.toHaveBeenCalled()
+    expect(mockLogger.error).not.toHaveBeenCalled()
+  })
+
+  it('should retry when operation fails and eventually succeed', async () => {
+    const mockOperation = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('first failure'))
+      .mockRejectedValueOnce(new Error('second failure'))
+      .mockResolvedValueOnce('success after retries')
+
+    const result = await withRetry(mockOperation, mockOptions)
+
+    expect(result).toBe('success after retries')
+    expect(mockOperation).toHaveBeenCalledTimes(3)
+    expect(mockLogger.warn).toHaveBeenCalledTimes(2)
+    expect(mockLogger.error).not.toHaveBeenCalled()
+  })
+
+  it('should throw an error after maximum retries are exhausted', async () => {
+    const mockError = new Error('persistent failure')
+    const mockOperation = jest.fn().mockRejectedValue(mockError)
+
+    await expect(withRetry(mockOperation, mockOptions)).rejects.toThrow('persistent failure')
+
+    expect(mockOperation).toHaveBeenCalledTimes(mockOptions.maxRetries)
+    expect(mockLogger.warn).toHaveBeenCalledTimes(mockOptions.maxRetries - 1)
+    expect(mockLogger.error).toHaveBeenCalledTimes(1)
+  })
+
+  it('should use default options when none are provided', async () => {
+    const mockOperation = jest.fn().mockResolvedValue('success')
+
+    const result = await withRetry(mockOperation)
+
+    expect(result).toBe('success')
+    expect(mockOperation).toHaveBeenCalledTimes(1)
+  })
+
+  it('should use provided operation name in log messages', async () => {
+    const mockOperation = jest.fn().mockRejectedValueOnce(new Error('failure')).mockResolvedValueOnce('success')
+
+    await withRetry(mockOperation, mockOptions, 'custom-operation')
+
+    expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('custom-operation failed'), expect.any(Object))
+  })
+
+  it('should apply jitter to delay calculation', async () => {
+    const mockOperation = jest.fn().mockRejectedValueOnce(new Error('failure')).mockResolvedValueOnce('success')
+
+    // Use Math.random mock to make jitter predictable
+    const mockRandom = jest.spyOn(Math, 'random').mockReturnValue(0.5)
+
+    await withRetry(mockOperation, { ...mockOptions, jitter: 0.2 })
+
+    // With Math.random = 0.5, jitter should be 0
+    // (since 0.5 * 2 - 1 = 0)
+    expect(mockLogger.warn).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({ delayMs: 10 }) // Should still be the initial delay
+    )
+
+    mockRandom.mockRestore()
+  })
+
+  it('should cap delay at maxDelayMs', async () => {
+    const mockOperation = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('failure 1'))
+      .mockRejectedValueOnce(new Error('failure 2'))
+      .mockRejectedValueOnce(new Error('failure 3'))
+      // Reduce the number of failures since we're only testing the delay calculation
+      // and not the full retry count
+      .mockResolvedValueOnce('success')
+
+    // Set a very high backoff factor to test capping
+    await withRetry(mockOperation, {
+      ...mockOptions,
+      initialDelayMs: 50,
+      maxDelayMs: 1000,
+      maxRetries: 4,
+      backoffFactor: 10 // Would normally go 50 -> 500 -> 5000, but should cap at 100
+    })
+
+    // First retry delay calculation: 50ms * 10 = 500ms (capped at 100ms)
+    expect(mockLogger.warn).toHaveBeenNthCalledWith(
+      1, // First warning call (for first retry)
+      expect.any(String),
+      expect.objectContaining({ delayMs: 50 }) // Should be capped at maxDelayMs
+    )
+
+    // Second retry delay would also be capped at 100ms
+    expect(mockLogger.warn).toHaveBeenNthCalledWith(
+      2, // Second warning call (for second retry)
+      expect.any(String),
+      expect.objectContaining({ delayMs: 500 })
+    )
+
+    expect(mockLogger.warn).toHaveBeenNthCalledWith(3, expect.any(String), expect.objectContaining({ delayMs: 1000 }))
+
+    // Function should have been called 4 times total
+    expect(mockOperation).toHaveBeenCalledTimes(4)
+  })
+})
+
+describe('createRetryableFunction', () => {
+  const mockLogger = {
+    warn: jest.fn(),
+    error: jest.fn(),
+    info: jest.fn()
+  }
+
+  const mockOptions: Partial<RetryOptions> = {
+    maxRetries: 2,
+    logger: mockLogger
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+  })
+
+  it('should create a function that applies retry logic', async () => {
+    const mockFn = jest.fn().mockRejectedValueOnce(new Error('first failure')).mockResolvedValueOnce('success')
+
+    const retryableFn = createRetryableFunction(mockFn, mockOptions)
+
+    const result = await retryableFn('arg1', 123)
+
+    expect(result).toBe('success')
+    expect(mockFn).toHaveBeenCalledTimes(2)
+    expect(mockFn).toHaveBeenCalledWith('arg1', 123)
+    expect(mockLogger.warn).toHaveBeenCalledTimes(1)
+  })
+
+  it('should pass through function parameters correctly', async () => {
+    const mockFn = jest.fn().mockResolvedValue('success')
+
+    const retryableFn = createRetryableFunction(mockFn, mockOptions)
+
+    await retryableFn('arg1', 123, { complex: true })
+
+    expect(mockFn).toHaveBeenCalledWith('arg1', 123, { complex: true })
+  })
+
+  it('should use custom operation name in logs', async () => {
+    const mockFn = jest.fn().mockRejectedValueOnce(new Error('failure')).mockResolvedValueOnce('success')
+
+    const retryableFn = createRetryableFunction(mockFn, mockOptions, 'custom-function')
+
+    await retryableFn()
+
+    expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('custom-function failed'), expect.any(Object))
+  })
+
+  it('should propagate the final error if all retries fail', async () => {
+    const mockError = new Error('persistent failure')
+    const mockFn = jest.fn().mockRejectedValue(mockError)
+
+    const retryableFn = createRetryableFunction(mockFn, mockOptions)
+
+    await expect(retryableFn()).rejects.toThrow('persistent failure')
+
+    expect(mockOptions.maxRetries).toBeDefined()
+    expect(mockFn).toHaveBeenCalledTimes(mockOptions.maxRetries ?? -1)
+  })
+})
+
+// Test for real-world timing scenarios
+describe('withRetry timing', () => {
+  // Restore original implementation for these tests
+  jest.unmock('../retry')
+
+  it('should respect actual delays between retries', async () => {
+    const startTime = Date.now()
+
+    const mockOperation = jest.fn().mockRejectedValueOnce(new Error('first failure')).mockResolvedValueOnce('success')
+
+    // Use smaller delays for faster tests
+    await withRetry(mockOperation, {
+      initialDelayMs: 50, // Start with 50ms
+      maxDelayMs: 1000,
+      maxRetries: 2,
+      backoffFactor: 2,
+      jitter: 0 // Disable jitter for predictable timing
+    })
+
+    const duration = Date.now() - startTime
+
+    // Should have waited approximately initialDelayMs
+    // Adding some margin for test environment variations
+    expect(duration).toBeGreaterThanOrEqual(40) // slightly less than initialDelayMs
+    expect(mockOperation).toHaveBeenCalledTimes(2)
+  }, 1000) // Timeout after 1 second
+})
+
+// Test with a decorated class
+describe('Using retry in class methods', () => {
+  class TestService {
+    counter = 0
+
+    async unstableFunction (): Promise<string> {
+      this.counter++
+      if (this.counter < 3) {
+        throw new Error(`Failure attempt ${this.counter}`)
+      }
+      return 'success'
+    }
+  }
+
+  it('should work with instance methods', async () => {
+    const service = new TestService()
+
+    // Create a retryable version of the method that's bound to the service
+    const retryableMethod = createRetryableFunction(service.unstableFunction.bind(service), { maxRetries: 3 })
+
+    const result = await retryableMethod()
+
+    expect(result).toBe('success')
+    expect(service.counter).toBe(3)
+  })
+})
+
+describe('withRetry with isRetryable option', () => {
+  // Create a mock logger to capture logs
+  const mockLogger = {
+    warn: jest.fn(),
+    error: jest.fn(),
+    info: jest.fn()
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    // Mock the sleep function to speed up tests
+    jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {
+      fn()
+      return 1 as any
+    })
+  })
+
+  it('should retry errors that are marked as retriable', async () => {
+    // Custom isRetryable function that only retries certain errors
+    const customRetriableCheck: IsRetryable = (err: any) => {
+      return err.message.includes('retriable')
+    }
+
+    const mockOperation = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('This is a retriable error'))
+      .mockRejectedValueOnce(new Error('This is a retriable error again'))
+      .mockResolvedValueOnce('success')
+
+    const result = await withRetry(mockOperation, {
+      maxRetries: 5,
+      logger: mockLogger,
+      isRetryable: customRetriableCheck
+    })
+
+    expect(result).toBe('success')
+    expect(mockOperation).toHaveBeenCalledTimes(3)
+    expect(mockLogger.warn).toHaveBeenCalledTimes(2)
+    expect(mockLogger.error).not.toHaveBeenCalled()
+  })
+
+  it('should not retry errors that are not marked as retriable', async () => {
+    // Custom isRetryable function that never retries
+    const neverRetry: IsRetryable = (_err: any) => {
+      return false
+    }
+
+    const mockOperation = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('This error should not be retried'))
+      .mockResolvedValueOnce('success')
+
+    await expect(
+      withRetry(mockOperation, {
+        maxRetries: 5,
+        logger: mockLogger,
+        isRetryable: neverRetry
+      })
+    ).rejects.toThrow('This error should not be retried')
+
+    expect(mockOperation).toHaveBeenCalledTimes(1)
+    expect(mockLogger.warn).not.toHaveBeenCalled()
+    expect(mockLogger.error).toHaveBeenCalledWith(
+      expect.stringContaining('failed with non-retriable error'),
+      expect.any(Object)
+    )
+  })
+
+  it('should have different behavior for different error types', async () => {
+    // Custom isRetryable function that retries only NetworkErrors
+    const retryOnlyNetworkErrors: IsRetryable = (err: any) => {
+      return err.name === 'NetworkError'
+    }
+
+    // Create different error types
+    const networkError = new Error('Network failed')
+    networkError.name = 'NetworkError'
+
+    const validationError = new Error('Validation failed')
+    validationError.name = 'ValidationError'
+
+    jest
+      .fn()
+      .mockRejectedValueOnce(networkError) // Should retry
+      .mockRejectedValueOnce(validationError) // Should not retry
+      .mockResolvedValueOnce('success')
+
+    // First test with network error - should be retried
+    const mockNetworkOp = jest.fn().mockRejectedValueOnce(networkError).mockResolvedValueOnce('network success')
+
+    const result1 = await withRetry(mockNetworkOp, {
+      maxRetries: 3,
+      logger: mockLogger,
+      isRetryable: retryOnlyNetworkErrors
+    })
+
+    expect(result1).toBe('network success')
+    expect(mockNetworkOp).toHaveBeenCalledTimes(2)
+
+    // Reset mocks
+    jest.clearAllMocks()
+
+    // Then test with validation error - should not be retried
+    const mockValidationOp = jest.fn().mockRejectedValueOnce(validationError)
+
+    await expect(
+      withRetry(mockValidationOp, {
+        maxRetries: 3,
+        logger: mockLogger,
+        isRetryable: retryOnlyNetworkErrors
+      })
+    ).rejects.toThrow('Validation failed')
+
+    expect(mockValidationOp).toHaveBeenCalledTimes(1)
+    expect(mockLogger.error).toHaveBeenCalledWith(expect.stringContaining('non-retriable error'), expect.any(Object))
+  })
+
+  it('should use the default retryAllErrors if isRetryable is not provided', async () => {
+    // All errors should be retried by default
+    const mockOperation = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('error 1'))
+      .mockRejectedValueOnce(new Error('error 2'))
+      .mockResolvedValueOnce('success')
+
+    const result = await withRetry(mockOperation, {
+      maxRetries: 5,
+      logger: mockLogger
+      // isRetryable not provided, should use default
+    })
+
+    expect(result).toBe('success')
+    expect(mockOperation).toHaveBeenCalledTimes(3)
+    expect(mockLogger.warn).toHaveBeenCalledTimes(2)
+  })
+
+  it('should pass the error to isRetryable function', async () => {
+    // Mock isRetryable function to track calls
+    const mockisRetryable = jest.fn().mockReturnValue(true)
+
+    const testError = new Error('test error')
+    const mockOperation = jest.fn().mockRejectedValueOnce(testError).mockResolvedValueOnce('success')
+
+    await withRetry(mockOperation, {
+      maxRetries: 3,
+      logger: mockLogger,
+      isRetryable: mockisRetryable
+    })
+
+    // Verify isRetryable was called with the actual error
+    expect(mockisRetryable).toHaveBeenCalledTimes(1)
+    expect(mockisRetryable).toHaveBeenCalledWith(testError)
+  })
+})
+
+describe('createRetryableFunction with isRetryable', () => {
+  const mockLogger = {
+    warn: jest.fn(),
+    error: jest.fn(),
+    info: jest.fn()
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    // Mock the sleep function to speed up tests
+    jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {
+      fn()
+      return 1 as any
+    })
+  })
+
+  it('should respect isRetryable when applied to a function', async () => {
+    // Function to wrap
+    const unstableFunction = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('retriable network error'))
+      .mockRejectedValueOnce(new Error('retriable network error'))
+      .mockResolvedValueOnce('success')
+
+    // Custom isRetryable that only retries network errors
+    const customRetriable: IsRetryable = (err: any) => {
+      return err.message.includes('network')
+    }
+
+    // Create retryable version with custom isRetryable
+    const retryableFunction = createRetryableFunction(
+      unstableFunction,
+      {
+        maxRetries: 3,
+        logger: mockLogger,
+        isRetryable: customRetriable
+      },
+      'custom-operation'
+    )
+
+    const result = await retryableFunction()
+    expect(result).toBe('success')
+  })
+})

utils/retry/src/__test__/retryable.test.ts

@@ -0,0 +1,158 @@
+//
+// Copyright © 2025 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { retryAllErrors, retryNetworkErrors } from '../retryable'
+
+describe('retryAllErrors', () => {
+  it('should return true for any error', () => {
+    expect(retryAllErrors(new Error('any error'))).toBe(true)
+    expect(retryAllErrors(new TypeError('type error'))).toBe(true)
+    expect(retryAllErrors(null)).toBe(true)
+    expect(retryAllErrors(undefined)).toBe(true)
+    expect(retryAllErrors({ custom: 'error object' })).toBe(true)
+    expect(retryAllErrors('error string')).toBe(true)
+  })
+})
+
+describe('retryNetworkErrors', () => {
+  it('should return false for null or undefined', () => {
+    expect(retryNetworkErrors(null)).toBe(false)
+    expect(retryNetworkErrors(undefined)).toBe(false)
+  })
+
+  it('should return true for errors with network-related names', () => {
+    const networkErrorNames = [
+      'NetworkError',
+      'FetchError',
+      'AbortError',
+      'TimeoutError',
+      'ConnectionError',
+      'ConnectionRefusedError',
+      'ETIMEDOUT',
+      'ECONNREFUSED',
+      'ECONNRESET',
+      'ENOTFOUND',
+      'EAI_AGAIN'
+    ]
+
+    networkErrorNames.forEach((name) => {
+      // Create an error with the specified name
+      const error = new Error('Test error')
+      error.name = name
+      expect(retryNetworkErrors(error)).toBe(true)
+    })
+  })
+
+  it('should return true for errors with network-related message patterns', () => {
+    const networkErrorMessages = [
+      'Network error occurred',
+      'Connection timed out',
+      'Connection refused',
+      'Connection reset',
+      'Socket hang up',
+      'DNS lookup failed',
+      'getaddrinfo ENOTFOUND api.example.com',
+      'connect ECONNREFUSED 127.0.0.1:8080',
+      'read ECONNRESET',
+      'connect ETIMEDOUT 192.168.1.1:443',
+      'getaddrinfo EAI_AGAIN myserver.local'
+    ]
+
+    networkErrorMessages.forEach((message) => {
+      expect(retryNetworkErrors(new Error(message))).toBe(true)
+    })
+  })
+
+  it('should return false for non-network related errors', () => {
+    const nonNetworkErrors = [
+      new Error('Invalid input'),
+      new TypeError('Cannot read property of undefined'),
+      new RangeError('Value out of range'),
+      new SyntaxError('Unexpected token'),
+      new Error('File not found'),
+      new Error('Permission denied'),
+      new Error('Invalid state')
+    ]
+
+    nonNetworkErrors.forEach((error) => {
+      expect(retryNetworkErrors(error)).toBe(false)
+    })
+  })
+
+  it('should return true for errors with server error status codes (5xx)', () => {
+    const serverErrors = [
+      createErrorWithStatus(500),
+      createErrorWithStatus(501),
+      createErrorWithStatus(502),
+      createErrorWithStatus(503),
+      createErrorWithStatus(504),
+      createErrorWithStatus(599)
+    ]
+
+    serverErrors.forEach((error) => {
+      expect(retryNetworkErrors(error)).toBe(true)
+    })
+  })
+
+  it('should return true for specific client error status codes', () => {
+    const retriableClientErrors = [
+      createErrorWithStatus(408), // Request Timeout
+      createErrorWithStatus(423), // Locked
+      createErrorWithStatus(425), // Too Early
+      createErrorWithStatus(429), // Too Many Requests
+      createErrorWithStatus(449) // Retry With
+    ]
+
+    retriableClientErrors.forEach((error) => {
+      expect(retryNetworkErrors(error)).toBe(true)
+    })
+  })
+
+  it('should return false for non-retriable client error status codes', () => {
+    const nonRetriableClientErrors = [
+      createErrorWithStatus(400), // Bad Request
+      createErrorWithStatus(401), // Unauthorized
+      createErrorWithStatus(403), // Forbidden
+      createErrorWithStatus(404), // Not Found
+      createErrorWithStatus(422) // Unprocessable Entity
+    ]
+
+    nonRetriableClientErrors.forEach((error) => {
+      expect(retryNetworkErrors(error)).toBe(false)
+    })
+  })
+
+  it('should return false for non-Error objects without network-related properties', () => {
+    const nonNetworkErrorObjects = [
+      { code: 'INVALID_INPUT' },
+      { code: 'AUTH_FAILED' },
+      { message: 'Invalid credentials' },
+      { error: 'Not found' }
+    ]
+
+    nonNetworkErrorObjects.forEach((errorObj) => {
+      expect(retryNetworkErrors(errorObj)).toBe(false)
+    })
+  })
+})
+
+/**
+ * Helper function to create an Error object with a status property
+ */
+function createErrorWithStatus (status: number): Error {
+  const error: any = new Error(`HTTP Error ${status}`)
+  error.status = status
+  return error
+}

utils/retry/src/decorator.ts

@@ -0,0 +1,35 @@
+//
+// Copyright © 2025 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+import { type RetryOptions, DEFAULT_RETRY_OPTIONS, withRetry } from './retry'
+
+/**
+ * Method decorator for adding retry functionality to class methods
+ *
+ * @param options - Retry configuration options
+ * @param operationName - Name of the operation for logging (defaults to method name)
+ * @returns Method decorator
+ */
+export function Retryable (options: Partial<RetryOptions> = DEFAULT_RETRY_OPTIONS): MethodDecorator {
+  return function (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) {
+    const originalMethod = descriptor.value as (...args: any[]) => any
+
+    descriptor.value = async function (...args: any[]) {
+      const methodName = propertyKey.toString()
+      return await withRetry(() => originalMethod.apply(this, args), options, methodName)
+    }
+
+    return descriptor
+  }
+}

utils/retry/src/index.ts

@@ -0,0 +1,18 @@
+//
+// Copyright © 2025 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+export * from './retry'
+export * from './decorator'
+export * from './retryable'

utils/retry/src/logger.ts

@@ -0,0 +1,31 @@
+//
+// Copyright © 2025 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+export interface Logger {
+  warn: (message: string, meta?: Record<string, any>) => void
+  error: (message: string, meta?: Record<string, any>) => void
+  info: (message: string, meta?: Record<string, any>) => void
+}
+
+export const defaultLogger: Logger = {
+  warn: (message: string, meta?: Record<string, any>) => {
+    console.warn(`[WARN] ${message}`, meta)
+  },
+  error: (message: string, meta?: Record<string, any>) => {
+    console.error(`[ERROR] ${message}`, meta)
+  },
+  info: (message: string, meta?: Record<string, any>) => {
+    console.info(`[INFO] ${message}`, meta)
+  }
+}

utils/retry/src/retry.ts

@@ -0,0 +1,146 @@
+//
+// Copyright © 2025 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+import { defaultLogger, type Logger } from './logger'
+import { type IsRetryable, retryAllErrors } from './retryable'
+
+/**
+ * Configuration options for the retry mechanism
+ */
+export interface RetryOptions {
+  /** Initial delay between retries in milliseconds */
+  initialDelayMs: number
+  /** Maximum delay between retries in milliseconds */
+  maxDelayMs: number
+  /** Maximum number of retry attempts */
+  maxRetries: number
+  /** Backoff factor for exponential delay increase */
+  backoffFactor: number
+  /** Function to determine if an error is retriable */
+  isRetryable: IsRetryable
+  /** Optional jitter factor (0-1) to add randomness to delay times */
+  jitter?: number
+  /** Logger to use (defaults to console logger) */
+  logger?: Logger
+}
+
+/**
+ * Default retry options
+ */
+export const DEFAULT_RETRY_OPTIONS: RetryOptions = {
+  initialDelayMs: 1000,
+  maxDelayMs: 30000,
+  maxRetries: 5,
+  backoffFactor: 1.5,
+  jitter: 0.2,
+  logger: defaultLogger,
+  isRetryable: retryAllErrors
+}
+
+/**
+ * Executes an operation with exponential backoff retry
+ *
+ * @param operation - Async operation to execute
+ * @param options - Retry configuration options
+ * @param operationName - Name of the operation for logging
+ * @returns The result of the operation
+ * @throws The last error encountered after all retries have been exhausted
+ */
+export async function withRetry<T> (
+  operation: () => Promise<T>,
+  options: Partial<RetryOptions> = {},
+  operationName: string = 'operation'
+): Promise<T> {
+  const config: RetryOptions = { ...DEFAULT_RETRY_OPTIONS, ...options }
+  const logger = config.logger ?? defaultLogger
+  let delayMs = config.initialDelayMs
+  let attempt = 1
+  let lastError: Error | undefined
+
+  while (attempt <= config.maxRetries) {
+    try {
+      return await operation()
+    } catch (err: any) {
+      lastError = err
+      const isLastAttempt = attempt >= config.maxRetries
+
+      if (isLastAttempt) {
+        logger.error(`${operationName} failed after ${attempt} attempts`, {
+          error: err.message,
+          attempt,
+          maxRetries: config.maxRetries
+        })
+        throw err
+      }
+      if (!config.isRetryable(err)) {
+        logger.error(`${operationName} failed with non-retriable error`, {
+          error: err.message,
+          attempt,
+          maxRetries: config.maxRetries
+        })
+        throw err
+      }
+
+      // Calculate next delay with jitter
+      let jitterAmount = 0
+      if (config.jitter != null && config.jitter > 0) {
+        jitterAmount = delayMs * config.jitter * (Math.random() * 2 - 1)
+      }
+      const actualDelay = Math.min(delayMs + jitterAmount, config.maxDelayMs)
+
+      logger.warn(`${operationName} failed, retrying in ${Math.round(actualDelay)}ms`, {
+        error: err.message,
+        attempt,
+        nextAttempt: attempt + 1,
+        delayMs: Math.round(actualDelay)
+      })
+
+      // Wait before retry
+      await sleep(actualDelay)
+
+      // Increase delay for next attempt (exponential backoff)
+      delayMs = Math.min(delayMs * config.backoffFactor, config.maxDelayMs)
+      attempt++
+    }
+  }
+
+  // This should not be reached due to the throw in the last iteration
+  throw lastError ?? new Error(`${operationName} failed for unknown reason`)
+}
+
+/**
+ * Promise-based sleep function
+ */
+function sleep (ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+/**
+ * Creates a retryable function from a base function
+ * Returns a wrapped function that will apply retry logic automatically
+ *
+ * @param fn - The function to make retryable
+ * @param options - Retry configuration options
+ * @param operationName - Name of the operation for logging
+ * @returns A wrapped function that applies retry logic
+ */
+export function createRetryableFunction<T extends (...args: any[]) => Promise<any>> (
+  fn: T,
+  options: Partial<RetryOptions> = {},
+  operationName: string = 'operation'
+): T {
+  return (async (...args: Parameters<T>): Promise<ReturnType<T>> => {
+    return await (withRetry(() => fn(...args), options, operationName) as Promise<ReturnType<T>>)
+  }) as T
+}

utils/retry/src/retryable.ts

@@ -0,0 +1,96 @@
+//
+// Copyright © 2025 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+export type IsRetryable = (error: Error | unknown) => boolean
+
+export const retryAllErrors: IsRetryable = (_error: Error | unknown): boolean => {
+  return true
+}
+
+const NETWORK_ERROR_NAMES = new Set([
+  'NetworkError',
+  'FetchError',
+  'AbortError',
+  'TimeoutError',
+  'ConnectionError',
+  'ConnectionRefusedError',
+  'ETIMEDOUT',
+  'ECONNREFUSED',
+  'ECONNRESET',
+  'ENOTFOUND',
+  'EAI_AGAIN'
+])
+
+/**
+ * Patterns in error messages that suggest network issues
+ */
+const NETWORK_ERROR_PATTERNS = [
+  /network/i,
+  /connection/i,
+  /timeout/i,
+  /unreachable/i,
+  /refused/i,
+  /reset/i,
+  /socket/i,
+  /DNS/i,
+  /ENOTFOUND/,
+  /ECONNREFUSED/,
+  /ECONNRESET/,
+  /ETIMEDOUT/,
+  /EAI_AGAIN/
+]
+
+/**
+ * Determine if an error is related to network issues
+ */
+export const retryNetworkErrors: IsRetryable = (error: Error | unknown): boolean => {
+  if (error == null) {
+    return false
+  }
+
+  // Check for error name
+  if (error instanceof Error) {
+    // Check if the error name is in our set of network errors
+    if (NETWORK_ERROR_NAMES.has(error.name)) {
+      return true
+    }
+
+    // Check if the error message matches our network error patterns
+    for (const pattern of NETWORK_ERROR_PATTERNS) {
+      if (pattern.test(error.message)) {
+        return true
+      }
+    }
+
+    // Check for status codes in response errors
+    if ('status' in error && typeof (error as any).status === 'number') {
+      const status = (error as any).status
+      // Retry server errors (5xx) and some specific client errors
+      return (
+        (status >= 500 && status < 600) || // Server errors
+        status === 429 || // Too Many Requests
+        status === 408 || // Request Timeout
+        status === 423 || // Locked
+        status === 425 || // Too Early
+        status === 449 || // Retry With
+        status === 503 || // Service Unavailable
+        status === 504 // Gateway Timeout
+      )
+    }
+  }
+
+  // If we couldn't identify it as a network error, don't retry
+  return false
+}

utils/retry/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json",
+
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./lib",
+    "declarationDir": "./types",
+    "tsBuildInfoFile": ".build/build.tsbuildinfo"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "lib", "dist", "types", "bundle"]
+}