mirror of
https://github.com/hcengineering/platform.git
synced 2025-06-09 09:20:54 +00:00
UBERF-11111: Add retry package (#9081)
* UBERF-11111: Add retry package Signed-off-by: Artem Savchenko <armisav@gmail.com> * UBERF-11111: Pass original error to logger Signed-off-by: Artem Savchenko <armisav@gmail.com> * UBERF-11111: Support different delay strategies Signed-off-by: Artem Savchenko <armisav@gmail.com> * UBERF-11111: Fix formatting in readme Signed-off-by: Artem Savchenko <armisav@gmail.com> --------- Signed-off-by: Artem Savchenko <armisav@gmail.com>
This commit is contained in:
parent
d81ef03105
commit
5ff2dafbc1
7
utils/retry/.eslintrc.js
Normal file
7
utils/retry/.eslintrc.js
Normal file
@ -0,0 +1,7 @@
|
||||
module.exports = {
|
||||
extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
|
||||
parserOptions: {
|
||||
tsconfigRootDir: __dirname,
|
||||
project: './tsconfig.json'
|
||||
}
|
||||
}
|
4
utils/retry/.npmignore
Normal file
4
utils/retry/.npmignore
Normal file
@ -0,0 +1,4 @@
|
||||
*
|
||||
!/lib/**
|
||||
!CHANGELOG.md
|
||||
/lib/**/__tests__/
|
5
utils/retry/config/rig.json
Normal file
5
utils/retry/config/rig.json
Normal file
@ -0,0 +1,5 @@
|
||||
{
|
||||
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
|
||||
"rigPackageName": "@hcengineering/platform-rig",
|
||||
"rigProfile": "node"
|
||||
}
|
7
utils/retry/jest.config.js
Normal file
7
utils/retry/jest.config.js
Normal file
@ -0,0 +1,7 @@
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
|
||||
roots: ["./src"],
|
||||
coverageReporters: ["text-summary", "html"]
|
||||
}
|
38
utils/retry/package.json
Normal file
38
utils/retry/package.json
Normal file
@ -0,0 +1,38 @@
|
||||
{
|
||||
"name": "@hcengineering/retry",
|
||||
"version": "0.6.0",
|
||||
"main": "lib/index.js",
|
||||
"svelte": "src/index.ts",
|
||||
"types": "types/index.d.ts",
|
||||
"author": "Copyright © Hardcore Engineering Inc.",
|
||||
"template": "@hcengineering/node-package",
|
||||
"license": "EPL-2.0",
|
||||
"scripts": {
|
||||
"build": "compile",
|
||||
"build:watch": "compile",
|
||||
"test": "jest --passWithNoTests --silent --forceExit",
|
||||
"format": "format src",
|
||||
"_phase:build": "compile transpile src",
|
||||
"_phase:test": "jest --passWithNoTests --silent --forceExit",
|
||||
"_phase:format": "format src",
|
||||
"_phase:validate": "compile validate"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@hcengineering/platform-rig": "^0.6.0",
|
||||
"@typescript-eslint/eslint-plugin": "^6.11.0",
|
||||
"eslint-plugin-import": "^2.26.0",
|
||||
"eslint-plugin-promise": "^6.1.1",
|
||||
"eslint-plugin-n": "^15.4.0",
|
||||
"eslint": "^8.54.0",
|
||||
"@typescript-eslint/parser": "^6.11.0",
|
||||
"eslint-config-standard-with-typescript": "^40.0.0",
|
||||
"prettier": "^3.1.0",
|
||||
"typescript": "^5.3.3",
|
||||
"@types/node": "~20.11.16",
|
||||
"jest": "^29.7.0",
|
||||
"ts-jest": "^29.1.1",
|
||||
"@types/jest": "^29.5.5"
|
||||
},
|
||||
"dependencies": {
|
||||
}
|
||||
}
|
252
utils/retry/readme.md
Normal file
252
utils/retry/readme.md
Normal file
@ -0,0 +1,252 @@
|
||||
# Retry Utility
|
||||
|
||||
A comprehensive TypeScript utility for handling transient failures with exponential backoff, jitter, and customizable retry conditions.
|
||||
|
||||
## Features
|
||||
|
||||
- ✅ **Exponential backoff** with configurable parameters
|
||||
- ✅ **Jitter support** to prevent thundering herd problems
|
||||
- ✅ **Customizable retry conditions** to control which errors should be retried
|
||||
- ✅ **TypeScript decorators** for clean, declarative retry logic
|
||||
- ✅ **Function wrappers** for retrofitting existing code with retry capabilities
|
||||
- ✅ **Comprehensive logging** of retry attempts and failures
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic Usage
|
||||
|
||||
Wrap any async operation with the `withRetry` function:
|
||||
|
||||
```typescript
|
||||
import { withRetry } from '@hcengineering/retry'
|
||||
|
||||
async function fetchData() {
|
||||
const data = await withRetry(
|
||||
async () => {
|
||||
// Your async operation that might fail transiently
|
||||
return await api.getData()
|
||||
},
|
||||
{ maxRetries: 3 }
|
||||
)
|
||||
return data
|
||||
}
|
||||
```
|
||||
|
||||
### Using Decorators
|
||||
|
||||
For class methods, you can use the `@Retryable` decorator for clean, declarative retry logic:
|
||||
|
||||
```typescript
|
||||
import { Retryable } from '@hcengineering/retry'
|
||||
|
||||
class UserService {
|
||||
@Retryable({ maxRetries: 5 })
|
||||
async getUserProfile(userId: string): Promise<UserProfile> {
|
||||
// This method will automatically retry on failure
|
||||
return await this.api.fetchUserProfile(userId)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Delay Strategies
|
||||
|
||||
The package provides several delay strategies to control the timing between retry attempts:
|
||||
|
||||
Exponential Backoff
|
||||
Increases the delay exponentially between retries, which is ideal for backing off from overloaded services:
|
||||
|
||||
```typescript
|
||||
import { withRetry, DelayStrategyFactory } from '@hcengineering/retry'
|
||||
|
||||
await withRetry(
|
||||
async () => await api.getData(),
|
||||
{
|
||||
maxRetries: 5,
|
||||
delayStrategy: DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 100, // Start with 100ms
|
||||
maxDelayMs: 10000, // Cap at 10 seconds
|
||||
backoffFactor: 2, // Double the delay each time (100, 200, 400, 800, 1600)
|
||||
jitter: 0.2 // Add ±20% randomness
|
||||
})
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
Fixed Delay
|
||||
Uses the same delay for all retry attempts, useful when retrying after a fixed cooldown period:
|
||||
```typescript
|
||||
import { withRetry, DelayStrategyFactory } from '@hcengineering/retry'
|
||||
|
||||
await withRetry(
|
||||
async () => await api.getData(),
|
||||
{
|
||||
maxRetries: 3,
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 1000, // Always wait 1 second between retries
|
||||
jitter: 0.1 // Optional: add ±10% randomness
|
||||
})
|
||||
}
|
||||
)
|
||||
```
|
||||
Fibonacci Delay
|
||||
Uses the Fibonacci sequence to calculate delays, providing a more moderate growth rate than exponential backoff:
|
||||
```typescript
|
||||
import { withRetry, DelayStrategyFactory } from '@hcengineering/retry'
|
||||
|
||||
await withRetry(
|
||||
async () => await api.getData(),
|
||||
{
|
||||
maxRetries: 6,
|
||||
delayStrategy: DelayStrategyFactory.fibonacci({
|
||||
baseDelayMs: 100, // Base unit for Fibonacci sequence
|
||||
maxDelayMs: 10000, // Maximum delay cap
|
||||
jitter: 0.2 // Add ±20% randomness
|
||||
})
|
||||
}
|
||||
)
|
||||
// Delays follow Fibonacci sequence: 100ms, 200ms, 300ms, 500ms, 800ms, ...
|
||||
```
|
||||
|
||||
### Custom Retry Conditions
|
||||
|
||||
You can specify which errors should trigger retries:
|
||||
|
||||
```typescript
|
||||
import { withRetry, retryNetworkErrors } from '@platform/utils/retry'
|
||||
|
||||
async function fetchData() {
|
||||
return await withRetry(
|
||||
async () => await api.getData(),
|
||||
{
|
||||
// Only retry network-related errors
|
||||
isRetryable: retryNetworkErrors,
|
||||
maxRetries: 5
|
||||
}
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
Create your own custom retry condition:
|
||||
|
||||
```typescript
|
||||
import { type IsRetryable } from '@platform/utils/retry'
|
||||
|
||||
// Custom retry condition
|
||||
const retryDatabaseErrors: IsRetryable = (error: unknown): boolean => {
|
||||
if (error instanceof DatabaseError) {
|
||||
// Only retry specific database errors
|
||||
return error.code === 'CONNECTION_LOST' ||
|
||||
error.code === 'DEADLOCK' ||
|
||||
error.code === 'TIMEOUT'
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Use it
|
||||
await withRetry(
|
||||
async () => await db.query('SELECT * FROM users'),
|
||||
{ isRetryable: retryDatabaseErrors }
|
||||
)
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `withRetry<T>(operation, options?, operationName?): Promise<T>`
|
||||
|
||||
Executes an async operation with retry logic.
|
||||
|
||||
- `operation`: `() => Promise<T>` - The async operation to execute
|
||||
- `options`: `Partial<RetryOptions>` - Retry configuration (optional)
|
||||
- `operationName`: `string` - Name for logging (optional)
|
||||
- Returns: `Promise<T>` - The result of the operation
|
||||
|
||||
### `createRetryableFunction<T>(fn, options?, operationName?): T`
|
||||
|
||||
Creates a retryable function from an existing function.
|
||||
|
||||
- `fn`: `T extends (...args: any[]) => Promise<any>` - The function to make retryable
|
||||
- `options`: `Partial<RetryOptions>` - Retry configuration (optional)
|
||||
- `operationName`: `string` - Name for logging (optional)
|
||||
- Returns: `T` - A wrapped function with retry logic
|
||||
|
||||
### `@Retryable(options?)`
|
||||
|
||||
Method decorator for adding retry functionality to class methods.
|
||||
|
||||
- `options`: `Partial<RetryOptions>` - Retry configuration (optional)
|
||||
|
||||
### RetryOptions
|
||||
|
||||
Configuration options for the retry mechanism:
|
||||
|
||||
| Option | Type | Default | Description |
|
||||
|--------|------|---------|-------------|
|
||||
| `initialDelayMs` | `number` | `1000` | Initial delay between retries in milliseconds |
|
||||
| `maxDelayMs` | `number` | `30000` | Maximum delay between retries in milliseconds |
|
||||
| `maxRetries` | `number` | `5` | Maximum number of retry attempts |
|
||||
| `backoffFactor` | `number` | `1.5` | Backoff factor for exponential delay increase |
|
||||
| `jitter` | `number` | `0.2` | Jitter factor (0-1) to add randomness to delay times |
|
||||
| `isRetryable` | `IsRetryable` | `retryAllErrors` | Function to determine if an error is retriable |
|
||||
| `logger` | `Logger` | `defaultLogger` | Logger to use |
|
||||
|
||||
### Retry Condition Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `retryAllErrors` | Retry on any error (default) |
|
||||
| `retryNetworkErrors` | Retry only on network-related errors |
|
||||
|
||||
## Examples
|
||||
|
||||
### Basic Retry with Custom Options
|
||||
|
||||
```typescript
|
||||
import { withRetry } from '@platform/utils/retry'
|
||||
|
||||
async function fetchDataWithRetry() {
|
||||
return await withRetry(
|
||||
async () => {
|
||||
const response = await fetch('https://api.example.com/data')
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error: ${response.status}`)
|
||||
}
|
||||
return await response.json()
|
||||
},
|
||||
{
|
||||
initialDelayMs: 300, // Start with 300ms delay
|
||||
maxDelayMs: 10000, // Max delay of 10 seconds
|
||||
maxRetries: 4, // Try up to 4 times (1 initial + 3 retries)
|
||||
backoffFactor: 2, // Double the delay each time
|
||||
jitter: 0.25 // Add 25% randomness to delay
|
||||
},
|
||||
'fetchApiData' // Name for logging
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
### Class with Multiple Retryable Methods
|
||||
|
||||
```typescript
|
||||
import { Retryable, retryNetworkErrors } from '@platform/utils/retry'
|
||||
|
||||
class DataService {
|
||||
@Retryable({
|
||||
maxRetries: 3,
|
||||
initialDelayMs: 200
|
||||
})
|
||||
async fetchUsers(): Promise<User[]> {
|
||||
// Will retry up to 3 times with initial 200ms delay
|
||||
return await this.api.getUsers()
|
||||
}
|
||||
|
||||
@Retryable({
|
||||
maxRetries: 5,
|
||||
initialDelayMs: 1000,
|
||||
isRetryable: retryNetworkErrors
|
||||
})
|
||||
async uploadFile(file: File): Promise<string> {
|
||||
// Will retry up to 5 times, but only for network errors
|
||||
return await this.api.uploadFile(file)
|
||||
}
|
||||
}
|
||||
```
|
388
utils/retry/src/__test__/decorator.test.ts
Normal file
388
utils/retry/src/__test__/decorator.test.ts
Normal file
@ -0,0 +1,388 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { DelayStrategyFactory } from '../delay'
|
||||
import { Retryable } from '../decorator'
|
||||
import { type RetryOptions } from '../retry'
|
||||
|
||||
// Instead of mocking withRetry, we'll mock setTimeout to avoid waiting in tests
|
||||
jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {
|
||||
fn()
|
||||
return 1 as any
|
||||
})
|
||||
|
||||
describe('Retryable decorator', () => {
|
||||
const mockLogger = {
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
info: jest.fn()
|
||||
}
|
||||
|
||||
// Update the mock options to use delay strategy
|
||||
const mockOptions: Partial<RetryOptions> = {
|
||||
maxRetries: 3,
|
||||
delayStrategy: DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 10,
|
||||
maxDelayMs: 100,
|
||||
backoffFactor: 2
|
||||
}),
|
||||
logger: mockLogger
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should retry failed operations', async () => {
|
||||
// Create a test class with decorated method that fails initially then succeeds
|
||||
const error = new Error('First attempt failed')
|
||||
class TestService {
|
||||
callCount = 0
|
||||
|
||||
@Retryable(mockOptions)
|
||||
async testMethod (param1: string, param2: number): Promise<string> {
|
||||
this.callCount++
|
||||
if (this.callCount === 1) {
|
||||
throw error
|
||||
}
|
||||
return `${param1}-${param2}`
|
||||
}
|
||||
}
|
||||
|
||||
const service = new TestService()
|
||||
const result = await service.testMethod('test', 123)
|
||||
|
||||
// Check results
|
||||
expect(result).toBe('test-123')
|
||||
expect(service.callCount).toBe(2) // Called once, failed, then succeeded on retry
|
||||
|
||||
// Check logs
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(1)
|
||||
expect(mockLogger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('testMethod failed'),
|
||||
expect.objectContaining({
|
||||
error,
|
||||
attempt: 1
|
||||
})
|
||||
)
|
||||
expect(mockLogger.error).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should work with default options', async () => {
|
||||
class TestService {
|
||||
callCount = 0
|
||||
|
||||
@Retryable()
|
||||
async testMethod (): Promise<string> {
|
||||
this.callCount++
|
||||
if (this.callCount === 1) {
|
||||
throw new Error('First attempt failed')
|
||||
}
|
||||
return 'success'
|
||||
}
|
||||
}
|
||||
|
||||
const service = new TestService()
|
||||
const result = await service.testMethod()
|
||||
|
||||
expect(result).toBe('success')
|
||||
expect(service.callCount).toBe(2) // Should have retried once
|
||||
})
|
||||
|
||||
it('should preserve class instance context (this)', async () => {
|
||||
class TestService {
|
||||
private counter = 0
|
||||
|
||||
@Retryable(mockOptions)
|
||||
async incrementAndGet (): Promise<number> {
|
||||
if (this.counter === 0) {
|
||||
this.counter++
|
||||
throw new Error('First attempt failed')
|
||||
}
|
||||
this.counter++
|
||||
return this.counter
|
||||
}
|
||||
|
||||
getCounter (): number {
|
||||
return this.counter
|
||||
}
|
||||
}
|
||||
|
||||
const service = new TestService()
|
||||
const result = await service.incrementAndGet()
|
||||
|
||||
// Check that the class context was preserved across retries
|
||||
expect(result).toBe(2)
|
||||
expect(service.getCounter()).toBe(2) // Incremented once per attempt
|
||||
})
|
||||
|
||||
it('should throw after max retries are exhausted', async () => {
|
||||
class TestService {
|
||||
@Retryable({
|
||||
maxRetries: 2, // Only try twice total (initial + 1 retry)
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 10
|
||||
}),
|
||||
logger: mockLogger
|
||||
})
|
||||
async alwaysFailingMethod (): Promise<string> {
|
||||
throw new Error('Persistent failure')
|
||||
}
|
||||
}
|
||||
|
||||
const service = new TestService()
|
||||
await expect(service.alwaysFailingMethod()).rejects.toThrow('Persistent failure')
|
||||
|
||||
// Should have tried twice in total (initial + 1 retry)
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(1)
|
||||
expect(mockLogger.error).toHaveBeenCalledTimes(1)
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('alwaysFailingMethod failed after 2 attempts'),
|
||||
expect.any(Object)
|
||||
)
|
||||
})
|
||||
|
||||
it('should handle async methods correctly', async () => {
|
||||
let resolutionCount = 0
|
||||
|
||||
// Create a class with an async method that fails then resolves
|
||||
class TestService {
|
||||
@Retryable(mockOptions)
|
||||
async delayedMethod (): Promise<string> {
|
||||
return await new Promise<string>((resolve, reject) => {
|
||||
resolutionCount++
|
||||
if (resolutionCount === 1) {
|
||||
reject(new Error('Delayed error'))
|
||||
} else {
|
||||
resolve('delayed success')
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const service = new TestService()
|
||||
const result = await service.delayedMethod()
|
||||
|
||||
expect(result).toBe('delayed success')
|
||||
expect(resolutionCount).toBe(2)
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('should retry according to the specified retry count', async () => {
|
||||
let callCount = 0
|
||||
|
||||
class TestService {
|
||||
@Retryable({
|
||||
maxRetries: 5, // Should try up to 5 times total
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 10
|
||||
}),
|
||||
logger: mockLogger
|
||||
})
|
||||
async unstableMethod (): Promise<string> {
|
||||
callCount++
|
||||
if (callCount < 4) {
|
||||
// Succeed on the 4th attempt
|
||||
throw new Error(`Failure #${callCount}`)
|
||||
}
|
||||
return 'success after retries'
|
||||
}
|
||||
}
|
||||
|
||||
const service = new TestService()
|
||||
const result = await service.unstableMethod()
|
||||
|
||||
expect(result).toBe('success after retries')
|
||||
expect(callCount).toBe(4) // Initial attempt + 3 retries = 4 total calls
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(3) // Should have logged 3 warnings
|
||||
})
|
||||
|
||||
it('should respect different delay strategies', async () => {
|
||||
// Override the setTimeout mock to capture delay values
|
||||
const delayValues: number[] = []
|
||||
jest.spyOn(global, 'setTimeout').mockImplementation((fn: any, delay: number | undefined) => {
|
||||
delayValues.push(delay ?? 0)
|
||||
fn()
|
||||
return 1 as any
|
||||
})
|
||||
|
||||
let callCount = 0
|
||||
|
||||
class TestService {
|
||||
@Retryable({
|
||||
maxRetries: 4,
|
||||
delayStrategy: DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 100,
|
||||
maxDelayMs: 500,
|
||||
backoffFactor: 2,
|
||||
jitter: 0 // Disable jitter for predictable tests
|
||||
})
|
||||
})
|
||||
async delayingMethod (): Promise<string> {
|
||||
callCount++
|
||||
if (callCount < 4) {
|
||||
throw new Error(`Attempt ${callCount} failed`)
|
||||
}
|
||||
return 'success'
|
||||
}
|
||||
}
|
||||
|
||||
const service = new TestService()
|
||||
await service.delayingMethod()
|
||||
|
||||
// Should have recorded 3 delays: initial, 2x initial, 4x initial (capped at maxDelayMs)
|
||||
expect(delayValues).toHaveLength(3)
|
||||
expect(delayValues[0]).toBe(100) // initial delay
|
||||
expect(delayValues[1]).toBe(200) // 2x initial
|
||||
expect(delayValues[2]).toBe(400) // 4x initial
|
||||
})
|
||||
|
||||
it('should test various delay strategies', async () => {
|
||||
// Override the setTimeout mock to capture delay values
|
||||
const delayValues: number[] = []
|
||||
jest.spyOn(global, 'setTimeout').mockImplementation((fn: any, delay: number | undefined) => {
|
||||
delayValues.push(delay ?? 0)
|
||||
fn()
|
||||
return 1 as any
|
||||
})
|
||||
|
||||
// Test Fixed strategy
|
||||
let callCount = 0
|
||||
class FixedTestService {
|
||||
@Retryable({
|
||||
maxRetries: 3,
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 100,
|
||||
jitter: 0 // Disable jitter for predictable tests
|
||||
})
|
||||
})
|
||||
async method (): Promise<string> {
|
||||
callCount++
|
||||
if (callCount < 3) {
|
||||
throw new Error(`Attempt ${callCount} failed`)
|
||||
}
|
||||
return 'success'
|
||||
}
|
||||
}
|
||||
|
||||
delayValues.length = 0 // Reset captured delays
|
||||
await new FixedTestService().method()
|
||||
expect(delayValues).toEqual([100, 100]) // Should be constant
|
||||
|
||||
// Test Fibonacci strategy
|
||||
callCount = 0
|
||||
class FibonacciTestService {
|
||||
@Retryable({
|
||||
maxRetries: 4,
|
||||
delayStrategy: DelayStrategyFactory.fibonacci({
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 1000,
|
||||
jitter: 0 // Disable jitter for predictable tests
|
||||
})
|
||||
})
|
||||
async method (): Promise<string> {
|
||||
callCount++
|
||||
if (callCount < 4) {
|
||||
throw new Error(`Attempt ${callCount} failed`)
|
||||
}
|
||||
return 'success'
|
||||
}
|
||||
}
|
||||
|
||||
delayValues.length = 0 // Reset captured delays
|
||||
await new FibonacciTestService().method()
|
||||
// Fibonacci sequence delay pattern
|
||||
expect(delayValues).toEqual([100, 200, 300]) // fib(2)=1, fib(3)=2, fib(4)=3 multiplied by baseDelayMs
|
||||
})
|
||||
|
||||
it('should handle methods returning non-promises', async () => {
|
||||
let callCount = 0
|
||||
|
||||
class TestService {
|
||||
@Retryable(mockOptions)
|
||||
nonAsyncMethod (input: string): string {
|
||||
callCount++
|
||||
if (callCount === 1) {
|
||||
throw new Error('Sync error')
|
||||
}
|
||||
return `processed-${input}`
|
||||
}
|
||||
}
|
||||
|
||||
const service = new TestService()
|
||||
// Even though the original method is not async, the decorated method returns a Promise
|
||||
// eslint-disable-next-line @typescript-eslint/await-thenable
|
||||
const result = await service.nonAsyncMethod('test')
|
||||
|
||||
expect(result).toBe('processed-test')
|
||||
expect(callCount).toBe(2) // Should have retried once
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('should handle static methods', async () => {
|
||||
let callCount = 0
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-extraneous-class
|
||||
class TestService {
|
||||
@Retryable(mockOptions)
|
||||
static async staticMethod (input: string): Promise<string> {
|
||||
callCount++
|
||||
if (callCount === 1) {
|
||||
throw new Error('Static method error')
|
||||
}
|
||||
return `static-${input}`
|
||||
}
|
||||
}
|
||||
|
||||
const result = await TestService.staticMethod('test')
|
||||
|
||||
expect(result).toBe('static-test')
|
||||
expect(callCount).toBe(2) // Should have retried once
|
||||
})
|
||||
|
||||
it('should respect isRetryable option', async () => {
|
||||
class TestService {
|
||||
callCount = 0
|
||||
|
||||
@Retryable({
|
||||
maxRetries: 3,
|
||||
delayStrategy: DelayStrategyFactory.fixed({ delayMs: 10 }),
|
||||
logger: mockLogger,
|
||||
isRetryable: (err) => {
|
||||
// Only retry errors with "retry" in the message
|
||||
return err instanceof Error && err.message.includes('Please retry')
|
||||
}
|
||||
})
|
||||
async conditionalRetryMethod (): Promise<string> {
|
||||
this.callCount++
|
||||
if (this.callCount === 1) {
|
||||
throw new Error('Please retry this') // should be retried
|
||||
}
|
||||
if (this.callCount === 2) {
|
||||
throw new Error('Do not retry this') // should not be retried
|
||||
}
|
||||
return 'success'
|
||||
}
|
||||
}
|
||||
|
||||
const service = new TestService()
|
||||
// Should fail with the second error since it won't be retried
|
||||
await expect(service.conditionalRetryMethod()).rejects.toThrow('Do not retry this')
|
||||
|
||||
expect(service.callCount).toBe(2) // Should have called twice (original + 1 retry)
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(1) // Only first error logged as warning
|
||||
expect(mockLogger.error).toHaveBeenCalledTimes(1) // Second error logged as error
|
||||
})
|
||||
})
|
296
utils/retry/src/__test__/delay.test.ts
Normal file
296
utils/retry/src/__test__/delay.test.ts
Normal file
@ -0,0 +1,296 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { DelayStrategyFactory, ExponentialBackoffStrategy, FibonacciDelayStrategy, FixedDelayStrategy } from '../delay'
|
||||
|
||||
describe('Delay Strategies', () => {
|
||||
// Mock Math.random to return fixed values in tests
|
||||
let randomMock: jest.SpyInstance
|
||||
|
||||
beforeEach(() => {
|
||||
// Default mock returns 0.5 for Math.random, which creates 0 jitter effect
|
||||
randomMock = jest.spyOn(Math, 'random').mockReturnValue(0.5)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
randomMock.mockRestore()
|
||||
})
|
||||
|
||||
describe('FixedDelayStrategy', () => {
|
||||
it('should return the same delay for all attempts without jitter', () => {
|
||||
const delay = 1000
|
||||
const strategy = new FixedDelayStrategy({ delayMs: delay })
|
||||
|
||||
expect(strategy.getDelay(1)).toBe(delay)
|
||||
expect(strategy.getDelay(2)).toBe(delay)
|
||||
expect(strategy.getDelay(5)).toBe(delay)
|
||||
expect(strategy.getDelay(10)).toBe(delay)
|
||||
})
|
||||
|
||||
it('should apply jitter correctly', () => {
|
||||
const delay = 1000
|
||||
const jitter = 0.2 // 20% jitter
|
||||
const strategy = new FixedDelayStrategy({ delayMs: delay, jitter })
|
||||
|
||||
// Mock random to return different values
|
||||
randomMock.mockReturnValueOnce(0.6) // => +0.2 * delay
|
||||
randomMock.mockReturnValueOnce(0.4) // => -0.2 * delay
|
||||
randomMock.mockReturnValueOnce(1.0) // => +1.0 * delay
|
||||
|
||||
// With 0.6 random value, jitter effect is (0.6-0.5)*2*0.2 = +0.04 => 4% increase
|
||||
expect(strategy.getDelay(1)).toBe(delay + delay * 0.2 * 0.2)
|
||||
|
||||
// With 0.4 random value, jitter effect is (0.4-0.5)*2*0.2 = -0.04 => 4% decrease
|
||||
expect(strategy.getDelay(2)).toBe(delay - delay * 0.2 * 0.2)
|
||||
|
||||
// With 1.0 random value, jitter effect is (1.0-0.5)*2*0.2 = +0.2 => 20% increase
|
||||
expect(strategy.getDelay(3)).toBe(delay + delay * 0.2 * 1.0)
|
||||
})
|
||||
|
||||
it('should never return negative values even with high jitter', () => {
|
||||
const delay = 100
|
||||
const jitter = 1.0 // 100% jitter, extreme case
|
||||
const strategy = new FixedDelayStrategy({ delayMs: delay, jitter })
|
||||
|
||||
// Mock random to return min value (full negative jitter)
|
||||
randomMock.mockReturnValue(0) // => -1.0 * delay
|
||||
|
||||
// With 0 random value and 1.0 jitter, the effect would be -100%, but should be capped at 0
|
||||
expect(strategy.getDelay(1)).toBe(0)
|
||||
})
|
||||
|
||||
it('should be creatable through factory method', () => {
|
||||
const strategy = DelayStrategyFactory.fixed({
|
||||
delayMs: 1000,
|
||||
jitter: 0.1
|
||||
})
|
||||
expect(strategy).toBeInstanceOf(FixedDelayStrategy)
|
||||
expect(strategy.getDelay(1)).toBe(1000) // With 0.5 mock random, no jitter effect
|
||||
})
|
||||
})
|
||||
|
||||
describe('ExponentialBackoffStrategy', () => {
|
||||
it('should increase delay exponentially without jitter', () => {
|
||||
const initial = 1000
|
||||
const max = 60000
|
||||
const factor = 2
|
||||
const strategy = new ExponentialBackoffStrategy({
|
||||
initialDelayMs: initial,
|
||||
maxDelayMs: max,
|
||||
backoffFactor: factor
|
||||
})
|
||||
|
||||
expect(strategy.getDelay(1)).toBe(initial) // 1000
|
||||
expect(strategy.getDelay(2)).toBe(initial * factor) // 2000
|
||||
expect(strategy.getDelay(3)).toBe(initial * Math.pow(factor, 2)) // 4000
|
||||
expect(strategy.getDelay(4)).toBe(initial * Math.pow(factor, 3)) // 8000
|
||||
})
|
||||
|
||||
it('should respect maximum delay', () => {
|
||||
const strategy = new ExponentialBackoffStrategy({
|
||||
initialDelayMs: 1000,
|
||||
maxDelayMs: 5000,
|
||||
backoffFactor: 2
|
||||
})
|
||||
|
||||
expect(strategy.getDelay(1)).toBe(1000)
|
||||
expect(strategy.getDelay(2)).toBe(2000)
|
||||
expect(strategy.getDelay(3)).toBe(4000)
|
||||
// Should be capped at 5000
|
||||
expect(strategy.getDelay(4)).toBe(5000)
|
||||
expect(strategy.getDelay(5)).toBe(5000)
|
||||
})
|
||||
|
||||
it('should apply jitter correctly', () => {
|
||||
const strategy = new ExponentialBackoffStrategy({
|
||||
initialDelayMs: 1000,
|
||||
maxDelayMs: 60000,
|
||||
backoffFactor: 2,
|
||||
jitter: 0.2
|
||||
})
|
||||
|
||||
// Mock random to return different values
|
||||
randomMock.mockReturnValueOnce(0.6) // => +0.2 * delay
|
||||
randomMock.mockReturnValueOnce(0.4) // => -0.2 * delay
|
||||
|
||||
// First attempt: 1000ms with jitter +4%
|
||||
expect(strategy.getDelay(1)).toBe(1000 + 1000 * 0.2 * 0.2)
|
||||
|
||||
// Second attempt: 2000ms with jitter -4%
|
||||
expect(strategy.getDelay(2)).toBe(2000 - 2000 * 0.2 * 0.2)
|
||||
})
|
||||
|
||||
it('should be creatable through factory method', () => {
|
||||
const strategy = DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 1000,
|
||||
maxDelayMs: 60000,
|
||||
backoffFactor: 2,
|
||||
jitter: 0.1
|
||||
})
|
||||
expect(strategy).toBeInstanceOf(ExponentialBackoffStrategy)
|
||||
expect(strategy.getDelay(1)).toBe(1000) // With 0.5 mock random, no jitter effect
|
||||
expect(strategy.getDelay(2)).toBe(2000)
|
||||
})
|
||||
})
|
||||
|
||||
describe('FibonacciDelayStrategy', () => {
|
||||
it('should follow Fibonacci sequence without jitter', () => {
|
||||
const baseDelay = 100
|
||||
const maxDelay = 10000
|
||||
const strategy = new FibonacciDelayStrategy({
|
||||
baseDelayMs: baseDelay,
|
||||
maxDelayMs: maxDelay
|
||||
})
|
||||
|
||||
expect(strategy.getDelay(1)).toBe(baseDelay * 1)
|
||||
expect(strategy.getDelay(2)).toBe(baseDelay * 2)
|
||||
expect(strategy.getDelay(3)).toBe(baseDelay * 3)
|
||||
expect(strategy.getDelay(4)).toBe(baseDelay * 5)
|
||||
expect(strategy.getDelay(5)).toBe(baseDelay * 8)
|
||||
expect(strategy.getDelay(6)).toBe(baseDelay * 13)
|
||||
})
|
||||
|
||||
it('should respect maximum delay', () => {
|
||||
const strategy = new FibonacciDelayStrategy({
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 500
|
||||
})
|
||||
|
||||
expect(strategy.getDelay(1)).toBe(100)
|
||||
expect(strategy.getDelay(2)).toBe(200)
|
||||
expect(strategy.getDelay(3)).toBe(300)
|
||||
expect(strategy.getDelay(4)).toBe(500)
|
||||
expect(strategy.getDelay(5)).toBe(500) // Capped at maxDelayMs
|
||||
expect(strategy.getDelay(6)).toBe(500) // Capped at maxDelayMs
|
||||
})
|
||||
|
||||
it('should cache Fibonacci calculations for performance', () => {
|
||||
const strategy = new FibonacciDelayStrategy({
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 10000
|
||||
})
|
||||
|
||||
// Access private cache to verify it's working
|
||||
const fibCache = (strategy as any).fibCache
|
||||
expect(fibCache.size).toBe(2) // Initial cache has 0->0 and 1->1
|
||||
|
||||
strategy.getDelay(7) // Should calculate fib(8) = 21
|
||||
|
||||
expect(fibCache.size).toBeGreaterThan(2)
|
||||
expect(fibCache.get(8)).toBe(21)
|
||||
})
|
||||
|
||||
it('should apply jitter correctly', () => {
|
||||
const strategy = new FibonacciDelayStrategy({
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 10000,
|
||||
jitter: 0.2
|
||||
})
|
||||
|
||||
// Mock random to return different values
|
||||
randomMock.mockReturnValueOnce(0.6) // => +0.2 * delay
|
||||
randomMock.mockReturnValueOnce(0.4) // => -0.2 * delay
|
||||
|
||||
// First attempt: 100ms (fib(2)=1 * 100) with jitter +4%
|
||||
expect(strategy.getDelay(1)).toBe(100 + 100 * 0.2 * 0.2)
|
||||
|
||||
// Second attempt: 100ms (fib(3)=1 * 100) with jitter -4%
|
||||
expect(strategy.getDelay(2)).toBe(200 - 200 * 0.2 * 0.2)
|
||||
})
|
||||
|
||||
it('should handle large Fibonacci numbers efficiently', () => {
|
||||
const strategy = new FibonacciDelayStrategy({
|
||||
baseDelayMs: 1,
|
||||
maxDelayMs: Number.MAX_SAFE_INTEGER
|
||||
})
|
||||
|
||||
// This would be extremely slow without memoization
|
||||
const start = performance.now()
|
||||
const delay = strategy.getDelay(40) // fib(41) = 165580141
|
||||
const duration = performance.now() - start
|
||||
|
||||
// Should be much faster than calculating naively
|
||||
expect(duration).toBeLessThan(100)
|
||||
expect(delay).toBe(165580141) // fib(41) * 1
|
||||
})
|
||||
|
||||
it('should be creatable through factory method', () => {
|
||||
const strategy = DelayStrategyFactory.fibonacci({
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 10000,
|
||||
jitter: 0.1
|
||||
})
|
||||
expect(strategy).toBeInstanceOf(FibonacciDelayStrategy)
|
||||
expect(strategy.getDelay(1)).toBe(100)
|
||||
expect(strategy.getDelay(2)).toBe(200)
|
||||
expect(strategy.getDelay(3)).toBe(300)
|
||||
expect(strategy.getDelay(4)).toBe(500)
|
||||
})
|
||||
})
|
||||
|
||||
describe('DelayStrategyFactory', () => {
|
||||
it('should create strategies with correct types', () => {
|
||||
expect(
|
||||
DelayStrategyFactory.fixed({
|
||||
delayMs: 1000
|
||||
})
|
||||
).toBeInstanceOf(FixedDelayStrategy)
|
||||
|
||||
expect(
|
||||
DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 1000,
|
||||
maxDelayMs: 60000,
|
||||
backoffFactor: 2
|
||||
})
|
||||
).toBeInstanceOf(ExponentialBackoffStrategy)
|
||||
|
||||
expect(
|
||||
DelayStrategyFactory.fibonacci({
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 10000
|
||||
})
|
||||
).toBeInstanceOf(FibonacciDelayStrategy)
|
||||
})
|
||||
|
||||
it('should pass parameters correctly to strategies', () => {
|
||||
const fixed = DelayStrategyFactory.fixed({
|
||||
delayMs: 1000,
|
||||
jitter: 0.1
|
||||
}) as FixedDelayStrategy
|
||||
expect((fixed as any).delayMs).toBe(1000)
|
||||
expect((fixed as any).jitter).toBe(0.1)
|
||||
|
||||
const exponential = DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 1000,
|
||||
maxDelayMs: 60000,
|
||||
backoffFactor: 2.5,
|
||||
jitter: 0.2
|
||||
}) as ExponentialBackoffStrategy
|
||||
expect((exponential as any).initialDelayMs).toBe(1000)
|
||||
expect((exponential as any).maxDelayMs).toBe(60000)
|
||||
expect((exponential as any).backoffFactor).toBe(2.5)
|
||||
expect((exponential as any).jitter).toBe(0.2)
|
||||
|
||||
const fibonacci = DelayStrategyFactory.fibonacci({
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 10000,
|
||||
jitter: 0.15
|
||||
}) as FibonacciDelayStrategy
|
||||
expect((fibonacci as any).baseDelayMs).toBe(100)
|
||||
expect((fibonacci as any).maxDelayMs).toBe(10000)
|
||||
expect((fibonacci as any).jitter).toBe(0.15)
|
||||
})
|
||||
})
|
||||
})
|
556
utils/retry/src/__test__/retry.test.ts
Normal file
556
utils/retry/src/__test__/retry.test.ts
Normal file
@ -0,0 +1,556 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { DelayStrategyFactory } from '../delay'
|
||||
import { withRetry, createRetryableFunction, type RetryOptions } from '../retry'
|
||||
import { type IsRetryable, retryAllErrors } from '../retryable'
|
||||
|
||||
// Mock the sleep function to speed up tests
|
||||
jest.mock('../delay', () => {
|
||||
const originalModule = jest.requireActual('../delay')
|
||||
return {
|
||||
...originalModule,
|
||||
// Override the internal sleep function to resolve immediately
|
||||
sleep: jest.fn().mockImplementation(() => Promise.resolve())
|
||||
}
|
||||
})
|
||||
|
||||
describe('withRetry', () => {
|
||||
// Create a mock logger to capture logs
|
||||
const mockLogger = {
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
info: jest.fn()
|
||||
}
|
||||
|
||||
// Use the new delayStrategy option
|
||||
const mockOptions: Partial<RetryOptions> = {
|
||||
maxRetries: 3,
|
||||
delayStrategy: DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 10,
|
||||
maxDelayMs: 100,
|
||||
backoffFactor: 2,
|
||||
jitter: 0
|
||||
}),
|
||||
logger: mockLogger,
|
||||
isRetryable: retryAllErrors
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should return the result when operation succeeds on first try', async () => {
|
||||
const mockOperation = jest.fn().mockResolvedValue('success')
|
||||
|
||||
const result = await withRetry(mockOperation, mockOptions)
|
||||
|
||||
expect(result).toBe('success')
|
||||
expect(mockOperation).toHaveBeenCalledTimes(1)
|
||||
expect(mockLogger.warn).not.toHaveBeenCalled()
|
||||
expect(mockLogger.error).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should retry when operation fails and eventually succeed', async () => {
|
||||
const mockOperation = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('first failure'))
|
||||
.mockRejectedValueOnce(new Error('second failure'))
|
||||
.mockResolvedValueOnce('success after retries')
|
||||
|
||||
const result = await withRetry(mockOperation, mockOptions)
|
||||
|
||||
expect(result).toBe('success after retries')
|
||||
expect(mockOperation).toHaveBeenCalledTimes(3)
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(2)
|
||||
expect(mockLogger.error).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should throw an error after maximum retries are exhausted', async () => {
|
||||
const mockError = new Error('persistent failure')
|
||||
const mockOperation = jest.fn().mockRejectedValue(mockError)
|
||||
|
||||
await expect(withRetry(mockOperation, mockOptions)).rejects.toThrow('persistent failure')
|
||||
|
||||
expect(mockOperation).toHaveBeenCalledTimes(mockOptions.maxRetries ?? -1)
|
||||
expect(mockLogger.error).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('should use default options when none are provided', async () => {
|
||||
const mockOperation = jest.fn().mockResolvedValue('success')
|
||||
|
||||
const result = await withRetry(mockOperation)
|
||||
|
||||
expect(result).toBe('success')
|
||||
expect(mockOperation).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('should use provided operation name in log messages', async () => {
|
||||
const mockOperation = jest.fn().mockRejectedValueOnce(new Error('failure')).mockResolvedValueOnce('success')
|
||||
|
||||
await withRetry(mockOperation, mockOptions, 'custom-operation')
|
||||
|
||||
expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('custom-operation failed'), expect.any(Object))
|
||||
})
|
||||
|
||||
it('should apply jitter to delay calculation', async () => {
|
||||
const mockOperation = jest.fn().mockRejectedValueOnce(new Error('failure')).mockResolvedValueOnce('success')
|
||||
|
||||
// Use Math.random mock to make jitter predictable
|
||||
const mockRandom = jest.spyOn(Math, 'random').mockReturnValue(0.5)
|
||||
|
||||
// Create options with jitter enabled
|
||||
const jitterOptions = {
|
||||
...mockOptions,
|
||||
delayStrategy: DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 10,
|
||||
maxDelayMs: 100,
|
||||
backoffFactor: 2,
|
||||
jitter: 0.2
|
||||
})
|
||||
}
|
||||
|
||||
await withRetry(mockOperation, jitterOptions)
|
||||
|
||||
// With Math.random = 0.5, jitter should be 0
|
||||
// (since 0.5 * 2 - 1 = 0)
|
||||
expect(mockLogger.warn).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({ delayMs: 10 }) // Should still be the initial delay
|
||||
)
|
||||
|
||||
mockRandom.mockRestore()
|
||||
})
|
||||
|
||||
it('should respect maximum delay', async () => {
|
||||
const mockOperation = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('failure 1'))
|
||||
.mockRejectedValueOnce(new Error('failure 2'))
|
||||
.mockRejectedValueOnce(new Error('failure 3'))
|
||||
.mockResolvedValueOnce('success')
|
||||
|
||||
// Use high backoff factor to test maximum delay cap
|
||||
const maxDelayOptions = {
|
||||
maxRetries: 4,
|
||||
delayStrategy: DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 50,
|
||||
maxDelayMs: 1000,
|
||||
backoffFactor: 10 // Would normally go 50 -> 500 -> 5000, but should cap at 1000
|
||||
}),
|
||||
logger: mockLogger
|
||||
}
|
||||
|
||||
await withRetry(mockOperation, maxDelayOptions)
|
||||
|
||||
// Check that delays are correctly calculated and capped
|
||||
expect(mockLogger.warn).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.any(String),
|
||||
expect.objectContaining({ delayMs: expect.any(Number) })
|
||||
)
|
||||
|
||||
// Second retry delay
|
||||
expect(mockLogger.warn).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.any(String),
|
||||
expect.objectContaining({ delayMs: 500 }) // 50 * 10 = 500
|
||||
)
|
||||
|
||||
// Third retry delay (should be capped)
|
||||
expect(mockLogger.warn).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.any(String),
|
||||
expect.objectContaining({ delayMs: 1000 }) // 500 * 10 = 5000, capped at 1000
|
||||
)
|
||||
|
||||
// Function should have been called 4 times total
|
||||
expect(mockOperation).toHaveBeenCalledTimes(4)
|
||||
})
|
||||
|
||||
it('should work with different delay strategies', async () => {
|
||||
// Test with fixed delay
|
||||
const fixedDelayOperation = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('failure 1'))
|
||||
.mockRejectedValueOnce(new Error('failure 2'))
|
||||
.mockResolvedValueOnce('success')
|
||||
|
||||
const fixedDelayOptions = {
|
||||
maxRetries: 3,
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 200,
|
||||
jitter: 0
|
||||
}),
|
||||
logger: mockLogger
|
||||
}
|
||||
|
||||
await withRetry(fixedDelayOperation, fixedDelayOptions)
|
||||
|
||||
// Both retries should have the same delay
|
||||
expect(mockLogger.warn).toHaveBeenNthCalledWith(1, expect.any(String), expect.objectContaining({ delayMs: 200 }))
|
||||
expect(mockLogger.warn).toHaveBeenNthCalledWith(2, expect.any(String), expect.objectContaining({ delayMs: 200 }))
|
||||
|
||||
// Clear mocks for next test
|
||||
jest.clearAllMocks()
|
||||
|
||||
// Test with Fibonacci delay
|
||||
const fibonacciDelayOperation = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('failure 1'))
|
||||
.mockRejectedValueOnce(new Error('failure 2'))
|
||||
.mockResolvedValueOnce('success')
|
||||
|
||||
const fibonacciDelayOptions = {
|
||||
maxRetries: 3,
|
||||
delayStrategy: DelayStrategyFactory.fibonacci({
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 10000,
|
||||
jitter: 0
|
||||
}),
|
||||
logger: mockLogger
|
||||
}
|
||||
|
||||
await withRetry(fibonacciDelayOperation, fibonacciDelayOptions)
|
||||
|
||||
// Delays should follow Fibonacci sequence
|
||||
expect(mockLogger.warn).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.any(String),
|
||||
expect.objectContaining({ delayMs: 100 }) // fib(2) = 1 * 100
|
||||
)
|
||||
expect(mockLogger.warn).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.any(String),
|
||||
expect.objectContaining({ delayMs: 200 }) // fib(3) = 2 * 100
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('createRetryableFunction', () => {
|
||||
const mockLogger = {
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
info: jest.fn()
|
||||
}
|
||||
|
||||
const mockOptions: Partial<RetryOptions> = {
|
||||
maxRetries: 2,
|
||||
delayStrategy: DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 10,
|
||||
maxDelayMs: 100,
|
||||
backoffFactor: 2
|
||||
}),
|
||||
logger: mockLogger
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should create a function that applies retry logic', async () => {
|
||||
const mockFn = jest.fn().mockRejectedValueOnce(new Error('first failure')).mockResolvedValueOnce('success')
|
||||
|
||||
const retryableFn = createRetryableFunction(mockFn, mockOptions)
|
||||
|
||||
const result = await retryableFn('arg1', 123)
|
||||
|
||||
expect(result).toBe('success')
|
||||
expect(mockFn).toHaveBeenCalledTimes(2)
|
||||
expect(mockFn).toHaveBeenCalledWith('arg1', 123)
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('should pass through function parameters correctly', async () => {
|
||||
const mockFn = jest.fn().mockResolvedValue('success')
|
||||
|
||||
const retryableFn = createRetryableFunction(mockFn, mockOptions)
|
||||
|
||||
await retryableFn('arg1', 123, { complex: true })
|
||||
|
||||
expect(mockFn).toHaveBeenCalledWith('arg1', 123, { complex: true })
|
||||
})
|
||||
|
||||
it('should use custom operation name in logs', async () => {
|
||||
const mockFn = jest.fn().mockRejectedValueOnce(new Error('failure')).mockResolvedValueOnce('success')
|
||||
|
||||
const retryableFn = createRetryableFunction(mockFn, mockOptions, 'custom-function')
|
||||
|
||||
await retryableFn()
|
||||
|
||||
expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('custom-function failed'), expect.any(Object))
|
||||
})
|
||||
|
||||
it('should propagate the final error if all retries fail', async () => {
|
||||
const mockError = new Error('persistent failure')
|
||||
const mockFn = jest.fn().mockRejectedValue(mockError)
|
||||
|
||||
const retryableFn = createRetryableFunction(mockFn, mockOptions)
|
||||
|
||||
await expect(retryableFn()).rejects.toThrow('persistent failure')
|
||||
|
||||
expect(mockOptions.maxRetries).toBeDefined()
|
||||
expect(mockFn).toHaveBeenCalledTimes(mockOptions.maxRetries ?? -1)
|
||||
})
|
||||
})
|
||||
|
||||
// Test with a decorated class
|
||||
describe('Using retry in class methods', () => {
|
||||
class TestService {
|
||||
counter = 0
|
||||
|
||||
async unstableFunction (): Promise<string> {
|
||||
this.counter++
|
||||
if (this.counter < 3) {
|
||||
throw new Error(`Failure attempt ${this.counter}`)
|
||||
}
|
||||
return 'success'
|
||||
}
|
||||
}
|
||||
|
||||
it('should work with instance methods', async () => {
|
||||
const service = new TestService()
|
||||
|
||||
// Create a retryable version of the method that's bound to the service
|
||||
const retryableMethod = createRetryableFunction(service.unstableFunction.bind(service), {
|
||||
maxRetries: 3,
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 10
|
||||
})
|
||||
})
|
||||
|
||||
const result = await retryableMethod()
|
||||
|
||||
expect(result).toBe('success')
|
||||
expect(service.counter).toBe(3)
|
||||
})
|
||||
})
|
||||
|
||||
describe('withRetry with isRetryable option', () => {
|
||||
// Create a mock logger to capture logs
|
||||
const mockLogger = {
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
info: jest.fn()
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks()
|
||||
// Mock the sleep function to speed up tests
|
||||
jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {
|
||||
fn()
|
||||
return 1 as any
|
||||
})
|
||||
})
|
||||
|
||||
it('should retry errors that are marked as retriable', async () => {
|
||||
// Custom isRetryable function that only retries certain errors
|
||||
const customRetriable: IsRetryable = (err: any) => {
|
||||
return err.message.includes('retriable')
|
||||
}
|
||||
|
||||
const mockOperation = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('This is a retriable error'))
|
||||
.mockRejectedValueOnce(new Error('This is a retriable error again'))
|
||||
.mockResolvedValueOnce('success')
|
||||
|
||||
const result = await withRetry(mockOperation, {
|
||||
maxRetries: 5,
|
||||
logger: mockLogger,
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 10
|
||||
}),
|
||||
isRetryable: customRetriable
|
||||
})
|
||||
|
||||
expect(result).toBe('success')
|
||||
expect(mockOperation).toHaveBeenCalledTimes(3)
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(2)
|
||||
expect(mockLogger.error).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should not retry errors that are not marked as retriable', async () => {
|
||||
// Custom isRetryable function that never retries
|
||||
const neverRetry: IsRetryable = (_err: any) => {
|
||||
return false
|
||||
}
|
||||
|
||||
const mockOperation = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('This error should not be retried'))
|
||||
.mockResolvedValueOnce('success')
|
||||
|
||||
await expect(
|
||||
withRetry(mockOperation, {
|
||||
maxRetries: 5,
|
||||
logger: mockLogger,
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 10
|
||||
}),
|
||||
isRetryable: neverRetry
|
||||
})
|
||||
).rejects.toThrow('This error should not be retried')
|
||||
|
||||
expect(mockOperation).toHaveBeenCalledTimes(1)
|
||||
expect(mockLogger.warn).not.toHaveBeenCalled()
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('failed with non-retriable error'),
|
||||
expect.any(Object)
|
||||
)
|
||||
})
|
||||
|
||||
it('should have different behavior for different error types', async () => {
|
||||
// Custom isRetryable function that retries only NetworkErrors
|
||||
const retryOnlyNetworkErrors: IsRetryable = (err: any) => {
|
||||
return err.name === 'NetworkError'
|
||||
}
|
||||
|
||||
// Create different error types
|
||||
const networkError = new Error('Network failed')
|
||||
networkError.name = 'NetworkError'
|
||||
|
||||
const validationError = new Error('Validation failed')
|
||||
validationError.name = 'ValidationError'
|
||||
|
||||
// First test with network error - should be retried
|
||||
const mockNetworkOp = jest.fn().mockRejectedValueOnce(networkError).mockResolvedValueOnce('network success')
|
||||
|
||||
const result1 = await withRetry(mockNetworkOp, {
|
||||
maxRetries: 3,
|
||||
logger: mockLogger,
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 10
|
||||
}),
|
||||
isRetryable: retryOnlyNetworkErrors
|
||||
})
|
||||
|
||||
expect(result1).toBe('network success')
|
||||
expect(mockNetworkOp).toHaveBeenCalledTimes(2)
|
||||
|
||||
// Reset mocks
|
||||
jest.clearAllMocks()
|
||||
|
||||
// Then test with validation error - should not be retried
|
||||
const mockValidationOp = jest.fn().mockRejectedValueOnce(validationError)
|
||||
|
||||
await expect(
|
||||
withRetry(mockValidationOp, {
|
||||
maxRetries: 3,
|
||||
logger: mockLogger,
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 10
|
||||
}),
|
||||
isRetryable: retryOnlyNetworkErrors
|
||||
})
|
||||
).rejects.toThrow('Validation failed')
|
||||
|
||||
expect(mockValidationOp).toHaveBeenCalledTimes(1)
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(expect.stringContaining('non-retriable error'), expect.any(Object))
|
||||
})
|
||||
|
||||
it('should use the default retryAllErrors if isRetryable is not provided', async () => {
|
||||
// All errors should be retried by default
|
||||
const mockOperation = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('error 1'))
|
||||
.mockRejectedValueOnce(new Error('error 2'))
|
||||
.mockResolvedValueOnce('success')
|
||||
|
||||
const result = await withRetry(mockOperation, {
|
||||
maxRetries: 5,
|
||||
logger: mockLogger,
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 10
|
||||
})
|
||||
// isRetryable not provided, should use default
|
||||
})
|
||||
|
||||
expect(result).toBe('success')
|
||||
expect(mockOperation).toHaveBeenCalledTimes(3)
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('should pass the error to isRetryable function', async () => {
|
||||
// Mock isRetryable function to track calls
|
||||
const mockisRetryable = jest.fn().mockReturnValue(true)
|
||||
|
||||
const testError = new Error('test error')
|
||||
const mockOperation = jest.fn().mockRejectedValueOnce(testError).mockResolvedValueOnce('success')
|
||||
|
||||
await withRetry(mockOperation, {
|
||||
maxRetries: 3,
|
||||
logger: mockLogger,
|
||||
delayStrategy: DelayStrategyFactory.fixed({
|
||||
delayMs: 10
|
||||
}),
|
||||
isRetryable: mockisRetryable
|
||||
})
|
||||
|
||||
// Verify isRetryable was called with the actual error
|
||||
expect(mockisRetryable).toHaveBeenCalledTimes(1)
|
||||
expect(mockisRetryable).toHaveBeenCalledWith(testError)
|
||||
})
|
||||
})
|
||||
|
||||
describe('createRetryableFunction with isRetryable', () => {
|
||||
const mockLogger = {
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
info: jest.fn()
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks()
|
||||
// Mock the sleep function to speed up tests
|
||||
jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {
|
||||
fn()
|
||||
return 1 as any
|
||||
})
|
||||
})
|
||||
|
||||
it('should respect isRetryable when applied to a function', async () => {
|
||||
// Function to wrap
|
||||
const unstableFunction = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('retriable network error'))
|
||||
.mockRejectedValueOnce(new Error('retriable network error'))
|
||||
.mockResolvedValueOnce('success')
|
||||
|
||||
// Custom isRetryable that only retries network errors
|
||||
const customRetriable: IsRetryable = (err: any) => {
|
||||
return err.message.includes('network')
|
||||
}
|
||||
|
||||
// Create retryable version with custom isRetryable
|
||||
const retryableFunction = createRetryableFunction(
|
||||
unstableFunction,
|
||||
{
|
||||
maxRetries: 3,
|
||||
logger: mockLogger,
|
||||
delayStrategy: DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 10,
|
||||
maxDelayMs: 100,
|
||||
backoffFactor: 2
|
||||
}),
|
||||
isRetryable: customRetriable
|
||||
},
|
||||
'custom-operation'
|
||||
)
|
||||
|
||||
const result = await retryableFunction()
|
||||
expect(result).toBe('success')
|
||||
})
|
||||
})
|
158
utils/retry/src/__test__/retryable.test.ts
Normal file
158
utils/retry/src/__test__/retryable.test.ts
Normal file
@ -0,0 +1,158 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { retryAllErrors, retryNetworkErrors } from '../retryable'
|
||||
|
||||
describe('retryAllErrors', () => {
|
||||
it('should return true for any error', () => {
|
||||
expect(retryAllErrors(new Error('any error'))).toBe(true)
|
||||
expect(retryAllErrors(new TypeError('type error'))).toBe(true)
|
||||
expect(retryAllErrors(null)).toBe(true)
|
||||
expect(retryAllErrors(undefined)).toBe(true)
|
||||
expect(retryAllErrors({ custom: 'error object' })).toBe(true)
|
||||
expect(retryAllErrors('error string')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('retryNetworkErrors', () => {
|
||||
it('should return false for null or undefined', () => {
|
||||
expect(retryNetworkErrors(null)).toBe(false)
|
||||
expect(retryNetworkErrors(undefined)).toBe(false)
|
||||
})
|
||||
|
||||
it('should return true for errors with network-related names', () => {
|
||||
const networkErrorNames = [
|
||||
'NetworkError',
|
||||
'FetchError',
|
||||
'AbortError',
|
||||
'TimeoutError',
|
||||
'ConnectionError',
|
||||
'ConnectionRefusedError',
|
||||
'ETIMEDOUT',
|
||||
'ECONNREFUSED',
|
||||
'ECONNRESET',
|
||||
'ENOTFOUND',
|
||||
'EAI_AGAIN'
|
||||
]
|
||||
|
||||
networkErrorNames.forEach((name) => {
|
||||
// Create an error with the specified name
|
||||
const error = new Error('Test error')
|
||||
error.name = name
|
||||
expect(retryNetworkErrors(error)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
it('should return true for errors with network-related message patterns', () => {
|
||||
const networkErrorMessages = [
|
||||
'Network error occurred',
|
||||
'Connection timed out',
|
||||
'Connection refused',
|
||||
'Connection reset',
|
||||
'Socket hang up',
|
||||
'DNS lookup failed',
|
||||
'getaddrinfo ENOTFOUND api.example.com',
|
||||
'connect ECONNREFUSED 127.0.0.1:8080',
|
||||
'read ECONNRESET',
|
||||
'connect ETIMEDOUT 192.168.1.1:443',
|
||||
'getaddrinfo EAI_AGAIN myserver.local'
|
||||
]
|
||||
|
||||
networkErrorMessages.forEach((message) => {
|
||||
expect(retryNetworkErrors(new Error(message))).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
it('should return false for non-network related errors', () => {
|
||||
const nonNetworkErrors = [
|
||||
new Error('Invalid input'),
|
||||
new TypeError('Cannot read property of undefined'),
|
||||
new RangeError('Value out of range'),
|
||||
new SyntaxError('Unexpected token'),
|
||||
new Error('File not found'),
|
||||
new Error('Permission denied'),
|
||||
new Error('Invalid state')
|
||||
]
|
||||
|
||||
nonNetworkErrors.forEach((error) => {
|
||||
expect(retryNetworkErrors(error)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
it('should return true for errors with server error status codes (5xx)', () => {
|
||||
const serverErrors = [
|
||||
createErrorWithStatus(500),
|
||||
createErrorWithStatus(501),
|
||||
createErrorWithStatus(502),
|
||||
createErrorWithStatus(503),
|
||||
createErrorWithStatus(504),
|
||||
createErrorWithStatus(599)
|
||||
]
|
||||
|
||||
serverErrors.forEach((error) => {
|
||||
expect(retryNetworkErrors(error)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
it('should return true for specific client error status codes', () => {
|
||||
const retriableClientErrors = [
|
||||
createErrorWithStatus(408), // Request Timeout
|
||||
createErrorWithStatus(423), // Locked
|
||||
createErrorWithStatus(425), // Too Early
|
||||
createErrorWithStatus(429), // Too Many Requests
|
||||
createErrorWithStatus(449) // Retry With
|
||||
]
|
||||
|
||||
retriableClientErrors.forEach((error) => {
|
||||
expect(retryNetworkErrors(error)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
it('should return false for non-retriable client error status codes', () => {
|
||||
const nonRetriableClientErrors = [
|
||||
createErrorWithStatus(400), // Bad Request
|
||||
createErrorWithStatus(401), // Unauthorized
|
||||
createErrorWithStatus(403), // Forbidden
|
||||
createErrorWithStatus(404), // Not Found
|
||||
createErrorWithStatus(422) // Unprocessable Entity
|
||||
]
|
||||
|
||||
nonRetriableClientErrors.forEach((error) => {
|
||||
expect(retryNetworkErrors(error)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
it('should return false for non-Error objects without network-related properties', () => {
|
||||
const nonNetworkErrorObjects = [
|
||||
{ code: 'INVALID_INPUT' },
|
||||
{ code: 'AUTH_FAILED' },
|
||||
{ message: 'Invalid credentials' },
|
||||
{ error: 'Not found' }
|
||||
]
|
||||
|
||||
nonNetworkErrorObjects.forEach((errorObj) => {
|
||||
expect(retryNetworkErrors(errorObj)).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* Helper function to create an Error object with a status property
|
||||
*/
|
||||
function createErrorWithStatus (status: number): Error {
|
||||
const error: any = new Error(`HTTP Error ${status}`)
|
||||
error.status = status
|
||||
return error
|
||||
}
|
35
utils/retry/src/decorator.ts
Normal file
35
utils/retry/src/decorator.ts
Normal file
@ -0,0 +1,35 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
import { type RetryOptions, DEFAULT_RETRY_OPTIONS, withRetry } from './retry'
|
||||
|
||||
/**
|
||||
* Method decorator for adding retry functionality to class methods
|
||||
*
|
||||
* @param options - Retry configuration options
|
||||
* @param operationName - Name of the operation for logging (defaults to method name)
|
||||
* @returns Method decorator
|
||||
*/
|
||||
export function Retryable (options: Partial<RetryOptions> = DEFAULT_RETRY_OPTIONS): MethodDecorator {
|
||||
return function (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) {
|
||||
const originalMethod = descriptor.value as (...args: any[]) => any
|
||||
|
||||
descriptor.value = async function (...args: any[]) {
|
||||
const methodName = propertyKey.toString()
|
||||
return await withRetry(() => originalMethod.apply(this, args), options, methodName)
|
||||
}
|
||||
|
||||
return descriptor
|
||||
}
|
||||
}
|
171
utils/retry/src/delay.ts
Normal file
171
utils/retry/src/delay.ts
Normal file
@ -0,0 +1,171 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
export const DelayStrategyFactory = {
|
||||
/**
|
||||
* Create a fixed delay strategy
|
||||
*/
|
||||
fixed (options: FixedDelayOptions): DelayStrategy {
|
||||
return new FixedDelayStrategy(options)
|
||||
},
|
||||
|
||||
/**
|
||||
* Create an exponential backoff delay strategy
|
||||
*/
|
||||
exponentialBackoff (options: ExponentialBackoffOptions): DelayStrategy {
|
||||
return new ExponentialBackoffStrategy(options)
|
||||
},
|
||||
|
||||
/**
|
||||
* Create a Fibonacci delay strategy
|
||||
*/
|
||||
fibonacci (options: FibonacciDelayOptions): DelayStrategy {
|
||||
return new FibonacciDelayStrategy(options)
|
||||
}
|
||||
}
|
||||
|
||||
export interface DelayStrategy {
|
||||
getDelay: (attempt: number) => number
|
||||
}
|
||||
|
||||
export interface FixedDelayOptions {
|
||||
/** Delay between retries in milliseconds */
|
||||
delayMs: number
|
||||
/** Optional jitter factor (0-1) to add randomness to delay times */
|
||||
jitter?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Fixed delay strategy - uses the same delay for all retry attempts
|
||||
*/
|
||||
export class FixedDelayStrategy implements DelayStrategy {
|
||||
private readonly delayMs: number
|
||||
private readonly jitter: number
|
||||
|
||||
constructor (options: FixedDelayOptions) {
|
||||
this.delayMs = options.delayMs
|
||||
this.jitter = options.jitter ?? 0
|
||||
}
|
||||
|
||||
getDelay (_attempt: number): number {
|
||||
if (this.jitter > 0) {
|
||||
const jitterAmount = this.delayMs * this.jitter * (Math.random() * 2 - 1)
|
||||
return Math.max(0, this.delayMs + jitterAmount)
|
||||
}
|
||||
return this.delayMs
|
||||
}
|
||||
}
|
||||
|
||||
export interface ExponentialBackoffOptions {
|
||||
/** Initial delay between retries in milliseconds */
|
||||
initialDelayMs: number
|
||||
/** Maximum delay between retries in milliseconds */
|
||||
maxDelayMs: number
|
||||
/** Backoff factor for exponential delay increase */
|
||||
backoffFactor: number
|
||||
/** Optional jitter factor (0-1) to add randomness to delay times */
|
||||
jitter?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Exponential backoff delay strategy - increases delay exponentially with each attempt
|
||||
*/
|
||||
export class ExponentialBackoffStrategy implements DelayStrategy {
|
||||
private readonly initialDelayMs: number
|
||||
private readonly maxDelayMs: number
|
||||
private readonly backoffFactor: number
|
||||
private readonly jitter: number
|
||||
|
||||
constructor (options: ExponentialBackoffOptions) {
|
||||
this.initialDelayMs = options.initialDelayMs
|
||||
this.maxDelayMs = options.maxDelayMs
|
||||
this.backoffFactor = options.backoffFactor
|
||||
this.jitter = options.jitter ?? 0
|
||||
}
|
||||
|
||||
getDelay (attempt: number): number {
|
||||
const baseDelay = Math.min(this.initialDelayMs * Math.pow(this.backoffFactor, attempt - 1), this.maxDelayMs)
|
||||
|
||||
if (this.jitter > 0) {
|
||||
const jitterAmount = baseDelay * this.jitter * (Math.random() * 2 - 1)
|
||||
return Math.min(Math.max(0, baseDelay + jitterAmount), this.maxDelayMs)
|
||||
}
|
||||
|
||||
return baseDelay
|
||||
}
|
||||
}
|
||||
|
||||
export interface FibonacciDelayOptions {
|
||||
/** Base unit for calculating Fibonacci sequence in milliseconds */
|
||||
baseDelayMs: number
|
||||
/** Maximum delay between retries in milliseconds */
|
||||
maxDelayMs: number
|
||||
/** Optional jitter factor (0-1) to add randomness to delay times */
|
||||
jitter?: number
|
||||
}
|
||||
|
||||
export class FibonacciDelayStrategy implements DelayStrategy {
|
||||
private readonly baseDelayMs: number
|
||||
private readonly maxDelayMs: number
|
||||
private readonly jitter: number
|
||||
|
||||
// Cache for Fibonacci numbers to improve performance
|
||||
private readonly fibCache: Map<number, number>
|
||||
|
||||
constructor (options: FibonacciDelayOptions) {
|
||||
this.baseDelayMs = options.baseDelayMs
|
||||
this.maxDelayMs = options.maxDelayMs
|
||||
this.jitter = options.jitter ?? 0
|
||||
this.fibCache = new Map<number, number>([
|
||||
[0, 0],
|
||||
[1, 1]
|
||||
])
|
||||
}
|
||||
|
||||
private fibonacci (n: number): number {
|
||||
// Return from cache if available
|
||||
if (this.fibCache.has(n)) {
|
||||
return this.fibCache.get(n) as number
|
||||
}
|
||||
|
||||
if (n <= 1) {
|
||||
return n
|
||||
}
|
||||
|
||||
// Calculate using recursion with memoization
|
||||
const result = this.fibonacci(n - 1) + this.fibonacci(n - 2)
|
||||
this.fibCache.set(n, result)
|
||||
return result
|
||||
}
|
||||
|
||||
getDelay (attempt: number): number {
|
||||
const fibNumber = this.fibonacci(attempt + 1)
|
||||
const baseDelay = Math.min(fibNumber * this.baseDelayMs, this.maxDelayMs)
|
||||
|
||||
if (this.jitter > 0) {
|
||||
const jitterAmount = baseDelay * this.jitter * (Math.random() * 2 - 1)
|
||||
return Math.min(Math.max(0, baseDelay + jitterAmount), this.maxDelayMs)
|
||||
}
|
||||
|
||||
return baseDelay
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Promise-based sleep function
|
||||
*/
|
||||
export function sleep (ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
18
utils/retry/src/index.ts
Normal file
18
utils/retry/src/index.ts
Normal file
@ -0,0 +1,18 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
export * from './retry'
|
||||
export * from './decorator'
|
||||
export * from './retryable'
|
31
utils/retry/src/logger.ts
Normal file
31
utils/retry/src/logger.ts
Normal file
@ -0,0 +1,31 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
export interface Logger {
|
||||
warn: (message: string, meta?: Record<string, any>) => void
|
||||
error: (message: string, meta?: Record<string, any>) => void
|
||||
info: (message: string, meta?: Record<string, any>) => void
|
||||
}
|
||||
|
||||
export const defaultLogger: Logger = {
|
||||
warn: (message: string, meta?: Record<string, any>) => {
|
||||
console.warn(`[WARN] ${message}`, meta)
|
||||
},
|
||||
error: (message: string, meta?: Record<string, any>) => {
|
||||
console.error(`[ERROR] ${message}`, meta)
|
||||
},
|
||||
info: (message: string, meta?: Record<string, any>) => {
|
||||
console.info(`[INFO] ${message}`, meta)
|
||||
}
|
||||
}
|
128
utils/retry/src/retry.ts
Normal file
128
utils/retry/src/retry.ts
Normal file
@ -0,0 +1,128 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
import { defaultLogger, type Logger } from './logger'
|
||||
import { type IsRetryable, retryAllErrors } from './retryable'
|
||||
import { type DelayStrategy, DelayStrategyFactory, sleep } from './delay'
|
||||
|
||||
/**
|
||||
* Configuration options for the retry mechanism
|
||||
*/
|
||||
export interface RetryOptions {
|
||||
/** Maximum number of retry attempts */
|
||||
maxRetries: number
|
||||
/** Function to determine if an error is retriable */
|
||||
isRetryable: IsRetryable
|
||||
/** Strategy for calculating delay between retries */
|
||||
delayStrategy: DelayStrategy
|
||||
/** Logger to use (defaults to console logger) */
|
||||
logger?: Logger
|
||||
}
|
||||
|
||||
/**
|
||||
* Default retry options
|
||||
*/
|
||||
export const DEFAULT_RETRY_OPTIONS: RetryOptions = {
|
||||
maxRetries: 5,
|
||||
isRetryable: retryAllErrors,
|
||||
delayStrategy: DelayStrategyFactory.exponentialBackoff({
|
||||
initialDelayMs: 1000,
|
||||
maxDelayMs: 30000,
|
||||
backoffFactor: 1.5,
|
||||
jitter: 0.2
|
||||
}),
|
||||
logger: defaultLogger
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes an operation with exponential backoff retry
|
||||
*
|
||||
* @param operation - Async operation to execute
|
||||
* @param options - Retry configuration options
|
||||
* @param operationName - Name of the operation for logging
|
||||
* @returns The result of the operation
|
||||
* @throws The last error encountered after all retries have been exhausted
|
||||
*/
|
||||
export async function withRetry<T> (
|
||||
operation: () => Promise<T>,
|
||||
options: Partial<RetryOptions> = {},
|
||||
operationName: string = 'operation'
|
||||
): Promise<T> {
|
||||
const config: RetryOptions = { ...DEFAULT_RETRY_OPTIONS, ...options }
|
||||
const logger = config.logger ?? defaultLogger
|
||||
let attempt = 1
|
||||
let lastError: Error | undefined
|
||||
|
||||
while (attempt <= config.maxRetries) {
|
||||
try {
|
||||
return await operation()
|
||||
} catch (error: any) {
|
||||
lastError = error
|
||||
const isLastAttempt = attempt >= config.maxRetries
|
||||
|
||||
if (isLastAttempt) {
|
||||
logger.error(`${operationName} failed after ${attempt} attempts`, {
|
||||
error,
|
||||
attempt,
|
||||
maxRetries: config.maxRetries
|
||||
})
|
||||
throw error
|
||||
}
|
||||
if (!config.isRetryable(error)) {
|
||||
logger.error(`${operationName} failed with non-retriable error`, {
|
||||
error,
|
||||
attempt,
|
||||
maxRetries: config.maxRetries
|
||||
})
|
||||
throw error
|
||||
}
|
||||
|
||||
// Get delay for next attempt from strategy
|
||||
const delayMs = Math.round(config.delayStrategy.getDelay(attempt))
|
||||
|
||||
logger.warn(`${operationName} failed, retrying in ${delayMs}ms`, {
|
||||
error,
|
||||
attempt,
|
||||
nextAttempt: attempt + 1,
|
||||
delayMs
|
||||
})
|
||||
|
||||
// Wait before retry
|
||||
await sleep(delayMs)
|
||||
attempt++
|
||||
}
|
||||
}
|
||||
|
||||
// This should not be reached due to the throw in the last iteration
|
||||
throw lastError ?? new Error(`${operationName} failed for unknown reason`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a retryable function from a base function
|
||||
* Returns a wrapped function that will apply retry logic automatically
|
||||
*
|
||||
* @param fn - The function to make retryable
|
||||
* @param options - Retry configuration options
|
||||
* @param operationName - Name of the operation for logging
|
||||
* @returns A wrapped function that applies retry logic
|
||||
*/
|
||||
export function createRetryableFunction<T extends (...args: any[]) => Promise<any>> (
|
||||
fn: T,
|
||||
options: Partial<RetryOptions> = {},
|
||||
operationName: string = 'operation'
|
||||
): T {
|
||||
return (async (...args: Parameters<T>): Promise<ReturnType<T>> => {
|
||||
return await (withRetry(() => fn(...args), options, operationName) as Promise<ReturnType<T>>)
|
||||
}) as T
|
||||
}
|
96
utils/retry/src/retryable.ts
Normal file
96
utils/retry/src/retryable.ts
Normal file
@ -0,0 +1,96 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
export type IsRetryable = (error: Error | unknown) => boolean
|
||||
|
||||
export const retryAllErrors: IsRetryable = (_error: Error | unknown): boolean => {
|
||||
return true
|
||||
}
|
||||
|
||||
const NETWORK_ERROR_NAMES = new Set([
|
||||
'NetworkError',
|
||||
'FetchError',
|
||||
'AbortError',
|
||||
'TimeoutError',
|
||||
'ConnectionError',
|
||||
'ConnectionRefusedError',
|
||||
'ETIMEDOUT',
|
||||
'ECONNREFUSED',
|
||||
'ECONNRESET',
|
||||
'ENOTFOUND',
|
||||
'EAI_AGAIN'
|
||||
])
|
||||
|
||||
/**
|
||||
* Patterns in error messages that suggest network issues
|
||||
*/
|
||||
const NETWORK_ERROR_PATTERNS = [
|
||||
/network/i,
|
||||
/connection/i,
|
||||
/timeout/i,
|
||||
/unreachable/i,
|
||||
/refused/i,
|
||||
/reset/i,
|
||||
/socket/i,
|
||||
/DNS/i,
|
||||
/ENOTFOUND/,
|
||||
/ECONNREFUSED/,
|
||||
/ECONNRESET/,
|
||||
/ETIMEDOUT/,
|
||||
/EAI_AGAIN/
|
||||
]
|
||||
|
||||
/**
|
||||
* Determine if an error is related to network issues
|
||||
*/
|
||||
export const retryNetworkErrors: IsRetryable = (error: Error | unknown): boolean => {
|
||||
if (error == null) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for error name
|
||||
if (error instanceof Error) {
|
||||
// Check if the error name is in our set of network errors
|
||||
if (NETWORK_ERROR_NAMES.has(error.name)) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if the error message matches our network error patterns
|
||||
for (const pattern of NETWORK_ERROR_PATTERNS) {
|
||||
if (pattern.test(error.message)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Check for status codes in response errors
|
||||
if ('status' in error && typeof (error as any).status === 'number') {
|
||||
const status = (error as any).status
|
||||
// Retry server errors (5xx) and some specific client errors
|
||||
return (
|
||||
(status >= 500 && status < 600) || // Server errors
|
||||
status === 429 || // Too Many Requests
|
||||
status === 408 || // Request Timeout
|
||||
status === 423 || // Locked
|
||||
status === 425 || // Too Early
|
||||
status === 449 || // Retry With
|
||||
status === 503 || // Service Unavailable
|
||||
status === 504 // Gateway Timeout
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// If we couldn't identify it as a network error, don't retry
|
||||
return false
|
||||
}
|
12
utils/retry/tsconfig.json
Normal file
12
utils/retry/tsconfig.json
Normal file
@ -0,0 +1,12 @@
|
||||
{
|
||||
"extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json",
|
||||
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./lib",
|
||||
"declarationDir": "./types",
|
||||
"tsBuildInfoFile": ".build/build.tsbuildinfo"
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "lib", "dist", "types", "bundle"]
|
||||
}
|
Loading…
Reference in New Issue
Block a user