UBERF-11111: Support different delay strategies

Signed-off-by: Artem Savchenko <armisav@gmail.com>
This commit is contained in:
Artem Savchenko 2025-05-26 13:04:24 +07:00
parent 54b70e1a7e
commit 68287bfe0d
6 changed files with 769 additions and 112 deletions

View File

@ -40,7 +40,7 @@ For class methods, you can use the `@Retryable` decorator for clean, declarative
import { Retryable } from '@hcengineering/retry'
class UserService {
@Retryable({ maxRetries: 3, initialDelayMs: 500 })
@Retryable({ maxRetries: 5 })
async getUserProfile(userId: string): Promise<UserProfile> {
// This method will automatically retry on failure
return await this.api.fetchUserProfile(userId)
@ -48,6 +48,65 @@ class UserService {
}
```
### Delay Strategies
The package provides several delay strategies to control the timing between retry attempts:
Exponential Backoff
Increases the delay exponentially between retries, which is ideal for backing off from overloaded services:
```
import { withRetry, DelayStrategyFactory } from '@hcengineering/retry'
await withRetry(
async () => await api.getData(),
{
maxRetries: 5,
delayStrategy: DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 100, // Start with 100ms
maxDelayMs: 10000, // Cap at 10 seconds
backoffFactor: 2, // Double the delay each time (100, 200, 400, 800, 1600)
jitter: 0.2 // Add ±20% randomness
})
}
)
```
Fixed Delay
Uses the same delay for all retry attempts, useful when retrying after a fixed cooldown period:
```
import { withRetry, DelayStrategyFactory } from '@hcengineering/retry'
await withRetry(
async () => await api.getData(),
{
maxRetries: 3,
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 1000, // Always wait 1 second between retries
jitter: 0.1 // Optional: add ±10% randomness
})
}
)
```
Fibonacci Delay
Uses the Fibonacci sequence to calculate delays, providing a more moderate growth rate than exponential backoff:
```
import { withRetry, DelayStrategyFactory } from '@hcengineering/retry'
await withRetry(
async () => await api.getData(),
{
maxRetries: 6,
delayStrategy: DelayStrategyFactory.fibonacci({
baseDelayMs: 100, // Base unit for Fibonacci sequence
maxDelayMs: 10000, // Maximum delay cap
jitter: 0.2 // Add ±20% randomness
})
}
)
// Delays follow Fibonacci sequence: 100ms, 200ms, 300ms, 500ms, 800ms, ...
```
### Custom Retry Conditions
You can specify which errors should trigger retries:

View File

@ -13,6 +13,7 @@
// limitations under the License.
//
import { DelayStrategyFactory } from '../delay'
import { Retryable } from '../decorator'
import { type RetryOptions } from '../retry'
@ -29,11 +30,14 @@ describe('Retryable decorator', () => {
info: jest.fn()
}
// Update the mock options to use delay strategy
const mockOptions: Partial<RetryOptions> = {
maxRetries: 3,
delayStrategy: DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 10,
maxDelayMs: 100,
maxRetries: 3,
backoffFactor: 2,
backoffFactor: 2
}),
logger: mockLogger
}
@ -127,8 +131,11 @@ describe('Retryable decorator', () => {
it('should throw after max retries are exhausted', async () => {
class TestService {
@Retryable({
...mockOptions,
maxRetries: 2 // Only try twice total (initial + 1 retry)
maxRetries: 2, // Only try twice total (initial + 1 retry)
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 10
}),
logger: mockLogger
})
async alwaysFailingMethod (): Promise<string> {
throw new Error('Persistent failure')
@ -179,7 +186,9 @@ describe('Retryable decorator', () => {
class TestService {
@Retryable({
maxRetries: 5, // Should try up to 5 times total
initialDelayMs: 10,
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 10
}),
logger: mockLogger
})
async unstableMethod (): Promise<string> {
@ -200,7 +209,7 @@ describe('Retryable decorator', () => {
expect(mockLogger.warn).toHaveBeenCalledTimes(3) // Should have logged 3 warnings
})
it('should respect different delay settings', async () => {
it('should respect different delay strategies', async () => {
// Override the setTimeout mock to capture delay values
const delayValues: number[] = []
jest.spyOn(global, 'setTimeout').mockImplementation((fn: any, delay: number | undefined) => {
@ -214,11 +223,13 @@ describe('Retryable decorator', () => {
class TestService {
@Retryable({
maxRetries: 4,
delayStrategy: DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 100,
maxDelayMs: 500,
backoffFactor: 2,
jitter: 0 // Disable jitter for predictable tests
})
})
async delayingMethod (): Promise<string> {
callCount++
if (callCount < 4) {
@ -238,6 +249,64 @@ describe('Retryable decorator', () => {
expect(delayValues[2]).toBe(400) // 4x initial
})
it('should test various delay strategies', async () => {
// Override the setTimeout mock to capture delay values
const delayValues: number[] = []
jest.spyOn(global, 'setTimeout').mockImplementation((fn: any, delay: number | undefined) => {
delayValues.push(delay ?? 0)
fn()
return 1 as any
})
// Test Fixed strategy
let callCount = 0
class FixedTestService {
@Retryable({
maxRetries: 3,
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 100,
jitter: 0 // Disable jitter for predictable tests
})
})
async method (): Promise<string> {
callCount++
if (callCount < 3) {
throw new Error(`Attempt ${callCount} failed`)
}
return 'success'
}
}
delayValues.length = 0 // Reset captured delays
await new FixedTestService().method()
expect(delayValues).toEqual([100, 100]) // Should be constant
// Test Fibonacci strategy
callCount = 0
class FibonacciTestService {
@Retryable({
maxRetries: 4,
delayStrategy: DelayStrategyFactory.fibonacci({
baseDelayMs: 100,
maxDelayMs: 1000,
jitter: 0 // Disable jitter for predictable tests
})
})
async method (): Promise<string> {
callCount++
if (callCount < 4) {
throw new Error(`Attempt ${callCount} failed`)
}
return 'success'
}
}
delayValues.length = 0 // Reset captured delays
await new FibonacciTestService().method()
// Fibonacci sequence delay pattern
expect(delayValues).toEqual([100, 200, 300]) // fib(2)=1, fib(3)=2, fib(4)=3 multiplied by baseDelayMs
})
it('should handle methods returning non-promises', async () => {
let callCount = 0
@ -288,7 +357,9 @@ describe('Retryable decorator', () => {
callCount = 0
@Retryable({
...mockOptions,
maxRetries: 3,
delayStrategy: DelayStrategyFactory.fixed({ delayMs: 10 }),
logger: mockLogger,
isRetryable: (err) => {
// Only retry errors with "retry" in the message
return err instanceof Error && err.message.includes('Please retry')

View File

@ -0,0 +1,296 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { DelayStrategyFactory, ExponentialBackoffStrategy, FibonacciDelayStrategy, FixedDelayStrategy } from '../delay'
describe('Delay Strategies', () => {
// Mock Math.random to return fixed values in tests
let randomMock: jest.SpyInstance
beforeEach(() => {
// Default mock returns 0.5 for Math.random, which creates 0 jitter effect
randomMock = jest.spyOn(Math, 'random').mockReturnValue(0.5)
})
afterEach(() => {
randomMock.mockRestore()
})
describe('FixedDelayStrategy', () => {
it('should return the same delay for all attempts without jitter', () => {
const delay = 1000
const strategy = new FixedDelayStrategy({ delayMs: delay })
expect(strategy.getDelay(1)).toBe(delay)
expect(strategy.getDelay(2)).toBe(delay)
expect(strategy.getDelay(5)).toBe(delay)
expect(strategy.getDelay(10)).toBe(delay)
})
it('should apply jitter correctly', () => {
const delay = 1000
const jitter = 0.2 // 20% jitter
const strategy = new FixedDelayStrategy({ delayMs: delay, jitter })
// Mock random to return different values
randomMock.mockReturnValueOnce(0.6) // => +0.2 * delay
randomMock.mockReturnValueOnce(0.4) // => -0.2 * delay
randomMock.mockReturnValueOnce(1.0) // => +1.0 * delay
// With 0.6 random value, jitter effect is (0.6-0.5)*2*0.2 = +0.04 => 4% increase
expect(strategy.getDelay(1)).toBe(delay + delay * 0.2 * 0.2)
// With 0.4 random value, jitter effect is (0.4-0.5)*2*0.2 = -0.04 => 4% decrease
expect(strategy.getDelay(2)).toBe(delay - delay * 0.2 * 0.2)
// With 1.0 random value, jitter effect is (1.0-0.5)*2*0.2 = +0.2 => 20% increase
expect(strategy.getDelay(3)).toBe(delay + delay * 0.2 * 1.0)
})
it('should never return negative values even with high jitter', () => {
const delay = 100
const jitter = 1.0 // 100% jitter, extreme case
const strategy = new FixedDelayStrategy({ delayMs: delay, jitter })
// Mock random to return min value (full negative jitter)
randomMock.mockReturnValue(0) // => -1.0 * delay
// With 0 random value and 1.0 jitter, the effect would be -100%, but should be capped at 0
expect(strategy.getDelay(1)).toBe(0)
})
it('should be creatable through factory method', () => {
const strategy = DelayStrategyFactory.fixed({
delayMs: 1000,
jitter: 0.1
})
expect(strategy).toBeInstanceOf(FixedDelayStrategy)
expect(strategy.getDelay(1)).toBe(1000) // With 0.5 mock random, no jitter effect
})
})
describe('ExponentialBackoffStrategy', () => {
it('should increase delay exponentially without jitter', () => {
const initial = 1000
const max = 60000
const factor = 2
const strategy = new ExponentialBackoffStrategy({
initialDelayMs: initial,
maxDelayMs: max,
backoffFactor: factor
})
expect(strategy.getDelay(1)).toBe(initial) // 1000
expect(strategy.getDelay(2)).toBe(initial * factor) // 2000
expect(strategy.getDelay(3)).toBe(initial * Math.pow(factor, 2)) // 4000
expect(strategy.getDelay(4)).toBe(initial * Math.pow(factor, 3)) // 8000
})
it('should respect maximum delay', () => {
const strategy = new ExponentialBackoffStrategy({
initialDelayMs: 1000,
maxDelayMs: 5000,
backoffFactor: 2
})
expect(strategy.getDelay(1)).toBe(1000)
expect(strategy.getDelay(2)).toBe(2000)
expect(strategy.getDelay(3)).toBe(4000)
// Should be capped at 5000
expect(strategy.getDelay(4)).toBe(5000)
expect(strategy.getDelay(5)).toBe(5000)
})
it('should apply jitter correctly', () => {
const strategy = new ExponentialBackoffStrategy({
initialDelayMs: 1000,
maxDelayMs: 60000,
backoffFactor: 2,
jitter: 0.2
})
// Mock random to return different values
randomMock.mockReturnValueOnce(0.6) // => +0.2 * delay
randomMock.mockReturnValueOnce(0.4) // => -0.2 * delay
// First attempt: 1000ms with jitter +4%
expect(strategy.getDelay(1)).toBe(1000 + 1000 * 0.2 * 0.2)
// Second attempt: 2000ms with jitter -4%
expect(strategy.getDelay(2)).toBe(2000 - 2000 * 0.2 * 0.2)
})
it('should be creatable through factory method', () => {
const strategy = DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 1000,
maxDelayMs: 60000,
backoffFactor: 2,
jitter: 0.1
})
expect(strategy).toBeInstanceOf(ExponentialBackoffStrategy)
expect(strategy.getDelay(1)).toBe(1000) // With 0.5 mock random, no jitter effect
expect(strategy.getDelay(2)).toBe(2000)
})
})
describe('FibonacciDelayStrategy', () => {
it('should follow Fibonacci sequence without jitter', () => {
const baseDelay = 100
const maxDelay = 10000
const strategy = new FibonacciDelayStrategy({
baseDelayMs: baseDelay,
maxDelayMs: maxDelay
})
expect(strategy.getDelay(1)).toBe(baseDelay * 1)
expect(strategy.getDelay(2)).toBe(baseDelay * 2)
expect(strategy.getDelay(3)).toBe(baseDelay * 3)
expect(strategy.getDelay(4)).toBe(baseDelay * 5)
expect(strategy.getDelay(5)).toBe(baseDelay * 8)
expect(strategy.getDelay(6)).toBe(baseDelay * 13)
})
it('should respect maximum delay', () => {
const strategy = new FibonacciDelayStrategy({
baseDelayMs: 100,
maxDelayMs: 500
})
expect(strategy.getDelay(1)).toBe(100)
expect(strategy.getDelay(2)).toBe(200)
expect(strategy.getDelay(3)).toBe(300)
expect(strategy.getDelay(4)).toBe(500)
expect(strategy.getDelay(5)).toBe(500) // Capped at maxDelayMs
expect(strategy.getDelay(6)).toBe(500) // Capped at maxDelayMs
})
it('should cache Fibonacci calculations for performance', () => {
const strategy = new FibonacciDelayStrategy({
baseDelayMs: 100,
maxDelayMs: 10000
})
// Access private cache to verify it's working
const fibCache = (strategy as any).fibCache
expect(fibCache.size).toBe(2) // Initial cache has 0->0 and 1->1
strategy.getDelay(7) // Should calculate fib(8) = 21
expect(fibCache.size).toBeGreaterThan(2)
expect(fibCache.get(8)).toBe(21)
})
it('should apply jitter correctly', () => {
const strategy = new FibonacciDelayStrategy({
baseDelayMs: 100,
maxDelayMs: 10000,
jitter: 0.2
})
// Mock random to return different values
randomMock.mockReturnValueOnce(0.6) // => +0.2 * delay
randomMock.mockReturnValueOnce(0.4) // => -0.2 * delay
// First attempt: 100ms (fib(2)=1 * 100) with jitter +4%
expect(strategy.getDelay(1)).toBe(100 + 100 * 0.2 * 0.2)
// Second attempt: 100ms (fib(3)=1 * 100) with jitter -4%
expect(strategy.getDelay(2)).toBe(200 - 200 * 0.2 * 0.2)
})
it('should handle large Fibonacci numbers efficiently', () => {
const strategy = new FibonacciDelayStrategy({
baseDelayMs: 1,
maxDelayMs: Number.MAX_SAFE_INTEGER
})
// This would be extremely slow without memoization
const start = performance.now()
const delay = strategy.getDelay(40) // fib(41) = 165580141
const duration = performance.now() - start
// Should be much faster than calculating naively
expect(duration).toBeLessThan(100)
expect(delay).toBe(165580141) // fib(41) * 1
})
it('should be creatable through factory method', () => {
const strategy = DelayStrategyFactory.fibonacci({
baseDelayMs: 100,
maxDelayMs: 10000,
jitter: 0.1
})
expect(strategy).toBeInstanceOf(FibonacciDelayStrategy)
expect(strategy.getDelay(1)).toBe(100)
expect(strategy.getDelay(2)).toBe(200)
expect(strategy.getDelay(3)).toBe(300)
expect(strategy.getDelay(4)).toBe(500)
})
})
describe('DelayStrategyFactory', () => {
it('should create strategies with correct types', () => {
expect(
DelayStrategyFactory.fixed({
delayMs: 1000
})
).toBeInstanceOf(FixedDelayStrategy)
expect(
DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 1000,
maxDelayMs: 60000,
backoffFactor: 2
})
).toBeInstanceOf(ExponentialBackoffStrategy)
expect(
DelayStrategyFactory.fibonacci({
baseDelayMs: 100,
maxDelayMs: 10000
})
).toBeInstanceOf(FibonacciDelayStrategy)
})
it('should pass parameters correctly to strategies', () => {
const fixed = DelayStrategyFactory.fixed({
delayMs: 1000,
jitter: 0.1
}) as FixedDelayStrategy
expect((fixed as any).delayMs).toBe(1000)
expect((fixed as any).jitter).toBe(0.1)
const exponential = DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 1000,
maxDelayMs: 60000,
backoffFactor: 2.5,
jitter: 0.2
}) as ExponentialBackoffStrategy
expect((exponential as any).initialDelayMs).toBe(1000)
expect((exponential as any).maxDelayMs).toBe(60000)
expect((exponential as any).backoffFactor).toBe(2.5)
expect((exponential as any).jitter).toBe(0.2)
const fibonacci = DelayStrategyFactory.fibonacci({
baseDelayMs: 100,
maxDelayMs: 10000,
jitter: 0.15
}) as FibonacciDelayStrategy
expect((fibonacci as any).baseDelayMs).toBe(100)
expect((fibonacci as any).maxDelayMs).toBe(10000)
expect((fibonacci as any).jitter).toBe(0.15)
})
})
})

View File

@ -13,12 +13,13 @@
// limitations under the License.
//
import { DelayStrategyFactory } from '../delay'
import { withRetry, createRetryableFunction, type RetryOptions } from '../retry'
import { type IsRetryable, retryAllErrors } from '../retryable'
// Mock the sleep function to speed up tests
jest.mock('../retry', () => {
const originalModule = jest.requireActual('../retry')
jest.mock('../delay', () => {
const originalModule = jest.requireActual('../delay')
return {
...originalModule,
// Override the internal sleep function to resolve immediately
@ -34,12 +35,15 @@ describe('withRetry', () => {
info: jest.fn()
}
const mockOptions: RetryOptions = {
// Use the new delayStrategy option
const mockOptions: Partial<RetryOptions> = {
maxRetries: 3,
delayStrategy: DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 10,
maxDelayMs: 100,
maxRetries: 3,
backoffFactor: 2,
jitter: 0,
jitter: 0
}),
logger: mockLogger,
isRetryable: retryAllErrors
}
@ -80,8 +84,7 @@ describe('withRetry', () => {
await expect(withRetry(mockOperation, mockOptions)).rejects.toThrow('persistent failure')
expect(mockOperation).toHaveBeenCalledTimes(mockOptions.maxRetries)
expect(mockLogger.warn).toHaveBeenCalledTimes(mockOptions.maxRetries - 1)
expect(mockOperation).toHaveBeenCalledTimes(mockOptions.maxRetries ?? -1)
expect(mockLogger.error).toHaveBeenCalledTimes(1)
})
@ -108,7 +111,18 @@ describe('withRetry', () => {
// Use Math.random mock to make jitter predictable
const mockRandom = jest.spyOn(Math, 'random').mockReturnValue(0.5)
await withRetry(mockOperation, { ...mockOptions, jitter: 0.2 })
// Create options with jitter enabled
const jitterOptions = {
...mockOptions,
delayStrategy: DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 10,
maxDelayMs: 100,
backoffFactor: 2,
jitter: 0.2
})
}
await withRetry(mockOperation, jitterOptions)
// With Math.random = 0.5, jitter should be 0
// (since 0.5 * 2 - 1 = 0)
@ -120,44 +134,109 @@ describe('withRetry', () => {
mockRandom.mockRestore()
})
it('should cap delay at maxDelayMs', async () => {
it('should respect maximum delay', async () => {
const mockOperation = jest
.fn()
.mockRejectedValueOnce(new Error('failure 1'))
.mockRejectedValueOnce(new Error('failure 2'))
.mockRejectedValueOnce(new Error('failure 3'))
// Reduce the number of failures since we're only testing the delay calculation
// and not the full retry count
.mockResolvedValueOnce('success')
// Set a very high backoff factor to test capping
await withRetry(mockOperation, {
...mockOptions,
// Use high backoff factor to test maximum delay cap
const maxDelayOptions = {
maxRetries: 4,
delayStrategy: DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 50,
maxDelayMs: 1000,
maxRetries: 4,
backoffFactor: 10 // Would normally go 50 -> 500 -> 5000, but should cap at 100
})
backoffFactor: 10 // Would normally go 50 -> 500 -> 5000, but should cap at 1000
}),
logger: mockLogger
}
// First retry delay calculation: 50ms * 10 = 500ms (capped at 100ms)
await withRetry(mockOperation, maxDelayOptions)
// Check that delays are correctly calculated and capped
expect(mockLogger.warn).toHaveBeenNthCalledWith(
1, // First warning call (for first retry)
1,
expect.any(String),
expect.objectContaining({ delayMs: 50 }) // Should be capped at maxDelayMs
expect.objectContaining({ delayMs: expect.any(Number) })
)
// Second retry delay would also be capped at 100ms
// Second retry delay
expect(mockLogger.warn).toHaveBeenNthCalledWith(
2, // Second warning call (for second retry)
2,
expect.any(String),
expect.objectContaining({ delayMs: 500 })
expect.objectContaining({ delayMs: 500 }) // 50 * 10 = 500
)
expect(mockLogger.warn).toHaveBeenNthCalledWith(3, expect.any(String), expect.objectContaining({ delayMs: 1000 }))
// Third retry delay (should be capped)
expect(mockLogger.warn).toHaveBeenNthCalledWith(
3,
expect.any(String),
expect.objectContaining({ delayMs: 1000 }) // 500 * 10 = 5000, capped at 1000
)
// Function should have been called 4 times total
expect(mockOperation).toHaveBeenCalledTimes(4)
})
it('should work with different delay strategies', async () => {
// Test with fixed delay
const fixedDelayOperation = jest
.fn()
.mockRejectedValueOnce(new Error('failure 1'))
.mockRejectedValueOnce(new Error('failure 2'))
.mockResolvedValueOnce('success')
const fixedDelayOptions = {
maxRetries: 3,
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 200,
jitter: 0
}),
logger: mockLogger
}
await withRetry(fixedDelayOperation, fixedDelayOptions)
// Both retries should have the same delay
expect(mockLogger.warn).toHaveBeenNthCalledWith(1, expect.any(String), expect.objectContaining({ delayMs: 200 }))
expect(mockLogger.warn).toHaveBeenNthCalledWith(2, expect.any(String), expect.objectContaining({ delayMs: 200 }))
// Clear mocks for next test
jest.clearAllMocks()
// Test with Fibonacci delay
const fibonacciDelayOperation = jest
.fn()
.mockRejectedValueOnce(new Error('failure 1'))
.mockRejectedValueOnce(new Error('failure 2'))
.mockResolvedValueOnce('success')
const fibonacciDelayOptions = {
maxRetries: 3,
delayStrategy: DelayStrategyFactory.fibonacci({
baseDelayMs: 100,
maxDelayMs: 10000,
jitter: 0
}),
logger: mockLogger
}
await withRetry(fibonacciDelayOperation, fibonacciDelayOptions)
// Delays should follow Fibonacci sequence
expect(mockLogger.warn).toHaveBeenNthCalledWith(
1,
expect.any(String),
expect.objectContaining({ delayMs: 100 }) // fib(2) = 1 * 100
)
expect(mockLogger.warn).toHaveBeenNthCalledWith(
2,
expect.any(String),
expect.objectContaining({ delayMs: 200 }) // fib(3) = 2 * 100
)
})
})
describe('createRetryableFunction', () => {
@ -169,6 +248,11 @@ describe('createRetryableFunction', () => {
const mockOptions: Partial<RetryOptions> = {
maxRetries: 2,
delayStrategy: DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 10,
maxDelayMs: 100,
backoffFactor: 2
}),
logger: mockLogger
}
@ -222,34 +306,6 @@ describe('createRetryableFunction', () => {
})
})
// Test for real-world timing scenarios
describe('withRetry timing', () => {
// Restore original implementation for these tests
jest.unmock('../retry')
it('should respect actual delays between retries', async () => {
const startTime = Date.now()
const mockOperation = jest.fn().mockRejectedValueOnce(new Error('first failure')).mockResolvedValueOnce('success')
// Use smaller delays for faster tests
await withRetry(mockOperation, {
initialDelayMs: 50, // Start with 50ms
maxDelayMs: 1000,
maxRetries: 2,
backoffFactor: 2,
jitter: 0 // Disable jitter for predictable timing
})
const duration = Date.now() - startTime
// Should have waited approximately initialDelayMs
// Adding some margin for test environment variations
expect(duration).toBeGreaterThanOrEqual(40) // slightly less than initialDelayMs
expect(mockOperation).toHaveBeenCalledTimes(2)
}, 1000) // Timeout after 1 second
})
// Test with a decorated class
describe('Using retry in class methods', () => {
class TestService {
@ -268,7 +324,12 @@ describe('Using retry in class methods', () => {
const service = new TestService()
// Create a retryable version of the method that's bound to the service
const retryableMethod = createRetryableFunction(service.unstableFunction.bind(service), { maxRetries: 3 })
const retryableMethod = createRetryableFunction(service.unstableFunction.bind(service), {
maxRetries: 3,
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 10
})
})
const result = await retryableMethod()
@ -296,7 +357,7 @@ describe('withRetry with isRetryable option', () => {
it('should retry errors that are marked as retriable', async () => {
// Custom isRetryable function that only retries certain errors
const customRetriableCheck: IsRetryable = (err: any) => {
const customRetriable: IsRetryable = (err: any) => {
return err.message.includes('retriable')
}
@ -309,7 +370,10 @@ describe('withRetry with isRetryable option', () => {
const result = await withRetry(mockOperation, {
maxRetries: 5,
logger: mockLogger,
isRetryable: customRetriableCheck
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 10
}),
isRetryable: customRetriable
})
expect(result).toBe('success')
@ -333,6 +397,9 @@ describe('withRetry with isRetryable option', () => {
withRetry(mockOperation, {
maxRetries: 5,
logger: mockLogger,
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 10
}),
isRetryable: neverRetry
})
).rejects.toThrow('This error should not be retried')
@ -358,18 +425,15 @@ describe('withRetry with isRetryable option', () => {
const validationError = new Error('Validation failed')
validationError.name = 'ValidationError'
jest
.fn()
.mockRejectedValueOnce(networkError) // Should retry
.mockRejectedValueOnce(validationError) // Should not retry
.mockResolvedValueOnce('success')
// First test with network error - should be retried
const mockNetworkOp = jest.fn().mockRejectedValueOnce(networkError).mockResolvedValueOnce('network success')
const result1 = await withRetry(mockNetworkOp, {
maxRetries: 3,
logger: mockLogger,
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 10
}),
isRetryable: retryOnlyNetworkErrors
})
@ -386,6 +450,9 @@ describe('withRetry with isRetryable option', () => {
withRetry(mockValidationOp, {
maxRetries: 3,
logger: mockLogger,
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 10
}),
isRetryable: retryOnlyNetworkErrors
})
).rejects.toThrow('Validation failed')
@ -404,7 +471,10 @@ describe('withRetry with isRetryable option', () => {
const result = await withRetry(mockOperation, {
maxRetries: 5,
logger: mockLogger
logger: mockLogger,
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 10
})
// isRetryable not provided, should use default
})
@ -423,6 +493,9 @@ describe('withRetry with isRetryable option', () => {
await withRetry(mockOperation, {
maxRetries: 3,
logger: mockLogger,
delayStrategy: DelayStrategyFactory.fixed({
delayMs: 10
}),
isRetryable: mockisRetryable
})
@ -467,6 +540,11 @@ describe('createRetryableFunction with isRetryable', () => {
{
maxRetries: 3,
logger: mockLogger,
delayStrategy: DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 10,
maxDelayMs: 100,
backoffFactor: 2
}),
isRetryable: customRetriable
},
'custom-operation'

171
utils/retry/src/delay.ts Normal file
View File

@ -0,0 +1,171 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export const DelayStrategyFactory = {
/**
* Create a fixed delay strategy
*/
fixed (options: FixedDelayOptions): DelayStrategy {
return new FixedDelayStrategy(options)
},
/**
* Create an exponential backoff delay strategy
*/
exponentialBackoff (options: ExponentialBackoffOptions): DelayStrategy {
return new ExponentialBackoffStrategy(options)
},
/**
* Create a Fibonacci delay strategy
*/
fibonacci (options: FibonacciDelayOptions): DelayStrategy {
return new FibonacciDelayStrategy(options)
}
}
export interface DelayStrategy {
getDelay: (attempt: number) => number
}
export interface FixedDelayOptions {
/** Delay between retries in milliseconds */
delayMs: number
/** Optional jitter factor (0-1) to add randomness to delay times */
jitter?: number
}
/**
* Fixed delay strategy - uses the same delay for all retry attempts
*/
export class FixedDelayStrategy implements DelayStrategy {
private readonly delayMs: number
private readonly jitter: number
constructor (options: FixedDelayOptions) {
this.delayMs = options.delayMs
this.jitter = options.jitter ?? 0
}
getDelay (_attempt: number): number {
if (this.jitter > 0) {
const jitterAmount = this.delayMs * this.jitter * (Math.random() * 2 - 1)
return Math.max(0, this.delayMs + jitterAmount)
}
return this.delayMs
}
}
export interface ExponentialBackoffOptions {
/** Initial delay between retries in milliseconds */
initialDelayMs: number
/** Maximum delay between retries in milliseconds */
maxDelayMs: number
/** Backoff factor for exponential delay increase */
backoffFactor: number
/** Optional jitter factor (0-1) to add randomness to delay times */
jitter?: number
}
/**
* Exponential backoff delay strategy - increases delay exponentially with each attempt
*/
export class ExponentialBackoffStrategy implements DelayStrategy {
private readonly initialDelayMs: number
private readonly maxDelayMs: number
private readonly backoffFactor: number
private readonly jitter: number
constructor (options: ExponentialBackoffOptions) {
this.initialDelayMs = options.initialDelayMs
this.maxDelayMs = options.maxDelayMs
this.backoffFactor = options.backoffFactor
this.jitter = options.jitter ?? 0
}
getDelay (attempt: number): number {
const baseDelay = Math.min(this.initialDelayMs * Math.pow(this.backoffFactor, attempt - 1), this.maxDelayMs)
if (this.jitter > 0) {
const jitterAmount = baseDelay * this.jitter * (Math.random() * 2 - 1)
return Math.min(Math.max(0, baseDelay + jitterAmount), this.maxDelayMs)
}
return baseDelay
}
}
export interface FibonacciDelayOptions {
/** Base unit for calculating Fibonacci sequence in milliseconds */
baseDelayMs: number
/** Maximum delay between retries in milliseconds */
maxDelayMs: number
/** Optional jitter factor (0-1) to add randomness to delay times */
jitter?: number
}
export class FibonacciDelayStrategy implements DelayStrategy {
private readonly baseDelayMs: number
private readonly maxDelayMs: number
private readonly jitter: number
// Cache for Fibonacci numbers to improve performance
private readonly fibCache: Map<number, number>
constructor (options: FibonacciDelayOptions) {
this.baseDelayMs = options.baseDelayMs
this.maxDelayMs = options.maxDelayMs
this.jitter = options.jitter ?? 0
this.fibCache = new Map<number, number>([
[0, 0],
[1, 1]
])
}
private fibonacci (n: number): number {
// Return from cache if available
if (this.fibCache.has(n)) {
return this.fibCache.get(n) as number
}
if (n <= 1) {
return n
}
// Calculate using recursion with memoization
const result = this.fibonacci(n - 1) + this.fibonacci(n - 2)
this.fibCache.set(n, result)
return result
}
getDelay (attempt: number): number {
const fibNumber = this.fibonacci(attempt + 1)
const baseDelay = Math.min(fibNumber * this.baseDelayMs, this.maxDelayMs)
if (this.jitter > 0) {
const jitterAmount = baseDelay * this.jitter * (Math.random() * 2 - 1)
return Math.min(Math.max(0, baseDelay + jitterAmount), this.maxDelayMs)
}
return baseDelay
}
}
/**
* Promise-based sleep function
*/
export function sleep (ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms))
}

View File

@ -14,23 +14,18 @@
//
import { defaultLogger, type Logger } from './logger'
import { type IsRetryable, retryAllErrors } from './retryable'
import { type DelayStrategy, DelayStrategyFactory, sleep } from './delay'
/**
* Configuration options for the retry mechanism
*/
export interface RetryOptions {
/** Initial delay between retries in milliseconds */
initialDelayMs: number
/** Maximum delay between retries in milliseconds */
maxDelayMs: number
/** Maximum number of retry attempts */
maxRetries: number
/** Backoff factor for exponential delay increase */
backoffFactor: number
/** Function to determine if an error is retriable */
isRetryable: IsRetryable
/** Optional jitter factor (0-1) to add randomness to delay times */
jitter?: number
/** Strategy for calculating delay between retries */
delayStrategy: DelayStrategy
/** Logger to use (defaults to console logger) */
logger?: Logger
}
@ -39,13 +34,15 @@ export interface RetryOptions {
* Default retry options
*/
export const DEFAULT_RETRY_OPTIONS: RetryOptions = {
maxRetries: 5,
isRetryable: retryAllErrors,
delayStrategy: DelayStrategyFactory.exponentialBackoff({
initialDelayMs: 1000,
maxDelayMs: 30000,
maxRetries: 5,
backoffFactor: 1.5,
jitter: 0.2,
logger: defaultLogger,
isRetryable: retryAllErrors
jitter: 0.2
}),
logger: defaultLogger
}
/**
@ -64,7 +61,6 @@ export async function withRetry<T> (
): Promise<T> {
const config: RetryOptions = { ...DEFAULT_RETRY_OPTIONS, ...options }
const logger = config.logger ?? defaultLogger
let delayMs = config.initialDelayMs
let attempt = 1
let lastError: Error | undefined
@ -92,25 +88,18 @@ export async function withRetry<T> (
throw error
}
// Calculate next delay with jitter
let jitterAmount = 0
if (config.jitter != null && config.jitter > 0) {
jitterAmount = delayMs * config.jitter * (Math.random() * 2 - 1)
}
const actualDelay = Math.min(delayMs + jitterAmount, config.maxDelayMs)
// Get delay for next attempt from strategy
const delayMs = Math.round(config.delayStrategy.getDelay(attempt))
logger.warn(`${operationName} failed, retrying in ${Math.round(actualDelay)}ms`, {
logger.warn(`${operationName} failed, retrying in ${delayMs}ms`, {
error,
attempt,
nextAttempt: attempt + 1,
delayMs: Math.round(actualDelay)
delayMs
})
// Wait before retry
await sleep(actualDelay)
// Increase delay for next attempt (exponential backoff)
delayMs = Math.min(delayMs * config.backoffFactor, config.maxDelayMs)
await sleep(delayMs)
attempt++
}
}
@ -119,13 +108,6 @@ export async function withRetry<T> (
throw lastError ?? new Error(`${operationName} failed for unknown reason`)
}
/**
* Promise-based sleep function
*/
function sleep (ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms))
}
/**
* Creates a retryable function from a base function
* Returns a wrapped function that will apply retry logic automatically