diff --git a/packages/browseros-agent/apps/agent/entrypoints/app/agents/useOpenClaw.ts b/packages/browseros-agent/apps/agent/entrypoints/app/agents/useOpenClaw.ts index 75b4f2266..8f3bd0025 100644 --- a/packages/browseros-agent/apps/agent/entrypoints/app/agents/useOpenClaw.ts +++ b/packages/browseros-agent/apps/agent/entrypoints/app/agents/useOpenClaw.ts @@ -87,6 +87,19 @@ async function clawFetch( message = body.error } } catch {} + + // Provide helpful context for rate limiting + if (res.status === 429) { + const retryAfter = res.headers.get('Retry-After') + const trimmedRetryAfter = retryAfter?.trim() + const retryAfterMessage = trimmedRetryAfter + ? /^\d+$/.test(trimmedRetryAfter) + ? `${Number(trimmedRetryAfter)}s` + : trimmedRetryAfter + : undefined + message = `Rate limited (429)${retryAfterMessage ? `: retry after ${retryAfterMessage}` : ''}. ${message}` + } + throw new Error(message) } return res.json() as Promise diff --git a/packages/browseros-agent/apps/server/src/lib/browseros-fetch.ts b/packages/browseros-agent/apps/server/src/lib/browseros-fetch.ts index 61bf1d827..28a5006c9 100644 --- a/packages/browseros-agent/apps/server/src/lib/browseros-fetch.ts +++ b/packages/browseros-agent/apps/server/src/lib/browseros-fetch.ts @@ -4,12 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-or-later * * Custom fetch for BrowserOS gateway requests. - * Adds X-BrowserOS-ID header for credit tracking, - * handles CREDITS_EXHAUSTED (429), and extracts OpenRouter-style error details. + * Features: + * - X-BrowserOS-ID header for credit tracking + * - Exponential backoff with jitter for rate-limited requests + * - Retry-After header respect (RFC 7231) + * - CREDITS_EXHAUSTED (429) detection (marked as non-retryable) + * - OpenRouter-style error detail extraction */ import { APICallError } from '@ai-sdk/provider' +import { HTTP_RETRY } from '@browseros/shared/constants/timeouts' import { logger } from './logger' +import { + createRetryableFetch, + type FetchLike, + isRetryableStatus, +} from './retry-handler' function resolveUrl(url: RequestInfo | URL): string { return typeof url === 'string' ? url : url.toString() @@ -39,11 +49,25 @@ function buildErrorMessage( } export function createBrowserOSFetch(browserosId: string): typeof fetch { - return (async (url: RequestInfo | URL, options?: RequestInit) => { + // Create a fetch wrapper that adds BrowserOS ID header + const fetchWithHeader: FetchLike = async ( + url: RequestInfo | URL, + options?: RequestInit, + ): Promise => { const headers = new Headers(options?.headers) headers.set('X-BrowserOS-ID', browserosId) + return globalThis.fetch(url, { ...options, headers }) + } - const response = await globalThis.fetch(url, { ...options, headers }) + // Wrap with retry handler for automatic exponential backoff + const retryableFetch = createRetryableFetch(fetchWithHeader, { + maxRetries: HTTP_RETRY.MAX_RETRIES, + baseDelayMs: HTTP_RETRY.BASE_DELAY_MS, + jitterMaxMs: HTTP_RETRY.JITTER_MAX_MS, + }) + + return (async (url: RequestInfo | URL, options?: RequestInit) => { + const response = await retryableFetch(url, options) const creditsRemaining = response.headers.get('X-Credits-Remaining') if (creditsRemaining !== null) { @@ -54,18 +78,28 @@ export function createBrowserOSFetch(browserosId: string): typeof fetch { const statusCode = response.status const responseBody = await response.text() const error = parseErrorBody(responseBody) + const isCreditsExhausted = error?.code === 'CREDITS_EXHAUSTED' + const retryAfterHeader = response.headers.get('Retry-After') + const responseHeaders = retryAfterHeader + ? { 'retry-after': retryAfterHeader } + : undefined - if (statusCode === 429 && error?.code === 'CREDITS_EXHAUSTED') { + // CREDITS_EXHAUSTED is fatal and should not be retried + if (statusCode === 429 && isCreditsExhausted) { throw new APICallError({ - message: error.message ?? 'Daily credits exhausted', + message: error?.message ?? 'Daily credits exhausted', url: resolveUrl(url), requestBodyValues: {}, statusCode, responseBody, + responseHeaders, isRetryable: false, }) } + // For other errors, calculate whether they're retryable + const isRetryable = isRetryableStatus(statusCode, false) + throw new APICallError({ message: error ? buildErrorMessage(statusCode, response.statusText, error) @@ -74,6 +108,8 @@ export function createBrowserOSFetch(browserosId: string): typeof fetch { requestBodyValues: {}, statusCode, responseBody, + responseHeaders, + isRetryable, }) } diff --git a/packages/browseros-agent/apps/server/src/lib/openrouter-fetch.ts b/packages/browseros-agent/apps/server/src/lib/openrouter-fetch.ts index d9ada38f3..00142431c 100644 --- a/packages/browseros-agent/apps/server/src/lib/openrouter-fetch.ts +++ b/packages/browseros-agent/apps/server/src/lib/openrouter-fetch.ts @@ -1,19 +1,31 @@ import { APICallError } from '@ai-sdk/provider' +import { HTTP_RETRY } from '@browseros/shared/constants/timeouts' +import { createRetryableFetch, isRetryableStatus } from './retry-handler' /** - * Creates a fetch function that extracts detailed error messages from OpenRouter-style APIs. + * Creates a fetch function for OpenRouter-compatible APIs. + * Features: + * - Exponential backoff with jitter for rate-limited requests (RFC 7231) + * - Retry-After header respect + * - OpenRouter-style error detail extraction and message enhancement * - * OpenRouter (and BrowserOS which uses it internally) wraps provider errors in a generic - * "Provider returned error" message, with actual details hidden in metadata.raw. - * This fetch intercepts HTTP errors and extracts the real error message. + * The Vercel AI SDK's retry mechanism is also applied by this fetch wrapper, + * providing defense-in-depth retry handling. This wrapper explicitly classifies + * retryability for the APICallError it throws, and the SDK also respects the + * status code and Retry-After headers when it applies its own retry logic. * - * IMPORTANT: Throws APICallError (not plain Error) so the Vercel AI SDK's retry mechanism - * works correctly. The SDK's APICallError automatically calculates `isRetryable` from - * the statusCode (408, 409, 429, 500+ are retryable) - we don't override this default. + * Note: This is primarily for non-gateway APIs. For gateway requests, use createBrowserOSFetch. */ export function createOpenRouterCompatibleFetch(): typeof fetch { + // Wrap with retry handler for exponential backoff + const retryableFetch = createRetryableFetch(globalThis.fetch, { + maxRetries: HTTP_RETRY.MAX_RETRIES, + baseDelayMs: HTTP_RETRY.BASE_DELAY_MS, + jitterMaxMs: HTTP_RETRY.JITTER_MAX_MS, + }) + return (async (url: RequestInfo | URL, options?: RequestInit) => { - const response = await globalThis.fetch(url, options) + const response = await retryableFetch(url, options) if (!response.ok) { const statusCode = response.status @@ -36,12 +48,21 @@ export function createOpenRouterCompatibleFetch(): typeof fetch { // Keep default error message if parsing fails } + // Determine if retryable based on status code + const isRetryable = isRetryableStatus(statusCode, false) + const retryAfterHeader = response.headers.get('Retry-After') + const responseHeaders = retryAfterHeader + ? { 'retry-after': retryAfterHeader } + : undefined + throw new APICallError({ message: errorMessage, url: typeof url === 'string' ? url : url.toString(), requestBodyValues: {}, statusCode, responseBody, + responseHeaders, + isRetryable, }) } diff --git a/packages/browseros-agent/apps/server/src/lib/retry-handler.ts b/packages/browseros-agent/apps/server/src/lib/retry-handler.ts new file mode 100644 index 000000000..1d731c0fc --- /dev/null +++ b/packages/browseros-agent/apps/server/src/lib/retry-handler.ts @@ -0,0 +1,373 @@ +/** + * @license + * Copyright 2025 BrowserOS + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * Production-quality HTTP retry handler with exponential backoff, jitter, and Retry-After support. + * Follows HTTP specifications for rate limiting (RFC 7231, RFC 6585). + */ + +import { HTTP_RETRY } from '@browseros/shared/constants/timeouts' +import { logger } from './logger' + +export interface RetryOptions { + /** + * Maximum number of retries (not including the initial attempt). + * @default HTTP_RETRY.MAX_RETRIES + */ + maxRetries?: number + /** + * Base delay in milliseconds for exponential backoff. + * @default HTTP_RETRY.BASE_DELAY_MS + */ + baseDelayMs?: number + /** + * Maximum jitter in milliseconds to add to each retry delay. + * @default HTTP_RETRY.JITTER_MAX_MS + */ + jitterMaxMs?: number + /** + * Custom status codes to retry on. If omitted, uses HTTP_RETRY.RETRYABLE_STATUS_CODES + */ + retryableStatusCodes?: readonly number[] +} + +export interface RetryContext { + url: string + statusCode: number + attempt: number + totalAttempts: number + retryAfterSeconds?: number + reason: string +} + +export type FetchLike = ( + url: RequestInfo | URL, + options?: RequestInit, +) => Promise + +/** + * Represents an error that occurred during request execution. + * Used internally to distinguish retry-eligible errors from fatal errors. + */ +export class RetryableError extends Error { + constructor( + message: string, + public readonly statusCode: number, + public readonly responseBody?: string, + public readonly retryAfterSeconds?: number, + ) { + super(message) + this.name = 'RetryableError' + } +} + +/** + * Parses the Retry-After header per HTTP specifications. + * Returns delay in seconds, or undefined if header is invalid. + * + * @param retryAfterHeader The value of the Retry-After header + * @returns Delay in seconds, or undefined if invalid + */ +export function parseRetryAfter( + retryAfterHeader: string | null | undefined, +): number | undefined { + const trimmedHeader = retryAfterHeader?.trim() + if (!trimmedHeader) return undefined + + // Attempt to parse as seconds (RFC 7231 Section 7.1.3) + if (/^\d+$/.test(trimmedHeader)) { + return Number(trimmedHeader) + } + + // Attempt to parse as HTTP-date (RFC 7231 Section 7.1.1) + // Example: "Fri, 31 Dec 1999 23:59:59 GMT" + try { + const retryDate = new Date(trimmedHeader) + if (!Number.isNaN(retryDate.getTime())) { + const delayMs = retryDate.getTime() - Date.now() + if (delayMs > 0) { + return Math.ceil(delayMs / 1000) + } + } + } catch { + // Invalid date format, will return undefined + } + + return undefined +} + +/** + * Calculates exponential backoff delay with jitter. + * Formula: baseDelay * 2^attempt + random(0, jitterMax) + * + * @param attempt Zero-indexed attempt number + * @param baseDelayMs Base delay in milliseconds + * @param jitterMaxMs Maximum jitter to add + * @returns Delay in milliseconds + */ +export function calculateBackoffDelay( + attempt: number, + baseDelayMs: number, + jitterMaxMs: number, +): number { + const exponentialDelay = baseDelayMs * 2 ** attempt + const jitter = Math.random() * jitterMaxMs + return exponentialDelay + jitter +} + +/** + * Determines if a status code is retryable. + * Non-retryable codes: 400, 401, 403, 404, and 429 with CREDITS_EXHAUSTED. + * + * @param statusCode HTTP status code + * @param isCreditsExhausted Whether the 429 error is due to exhausted credits + * @param retryableStatusCodes Custom list of retryable codes + * @returns Whether the error should be retried + */ +export function isRetryableStatus( + statusCode: number, + isCreditsExhausted = false, + retryableStatusCodes?: readonly number[], +): boolean { + // 429 with CREDITS_EXHAUSTED is NOT retryable + if (statusCode === 429 && isCreditsExhausted) return false + + // Use custom codes if provided, otherwise use default + const codes = retryableStatusCodes ?? HTTP_RETRY.RETRYABLE_STATUS_CODES + return codes.includes(statusCode) +} + +/** + * Handles a network error during fetch retry, either throwing (if max retries reached) or scheduling retry. + * @returns Delay in milliseconds to wait, or throws if max retries exhausted + */ +function handleNetworkError( + error: Error, + attempt: number, + maxRetries: number, + baseDelayMs: number, + jitterMaxMs: number, + urlStr: string, +): number { + if (attempt === maxRetries) { + logger.error('HTTP request failed after max retries (network error)', { + url: urlStr, + error: error instanceof Error ? error.message : String(error), + attempts: attempt + 1, + }) + throw error + } + + const backoffDelayMs = calculateBackoffDelay( + attempt, + baseDelayMs, + jitterMaxMs, + ) + logger.debug('HTTP request failed (network), will retry', { + url: urlStr, + error: error instanceof Error ? error.message : String(error), + attempt: attempt + 1, + nextRetryMs: backoffDelayMs, + }) + + return backoffDelayMs +} + +/** + * Cancels a response body so discarded retry responses do not keep streams open. + */ +async function cancelResponseBody(response: Response): Promise { + if (!response.body) return + + try { + await response.body.cancel() + } catch { + // Best effort only. + } +} + +/** + * Helper to wait for a given number of milliseconds. + * @param ms Milliseconds to wait + */ +async function wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +/** + * Evaluates a response to determine if it should be retried. + * Returns retry decision and delay (if retrying). + */ +async function evaluateResponseForRetry( + response: Response, + attempt: number, + maxRetries: number, + baseDelayMs: number, + jitterMaxMs: number, + retryableStatusCodes: readonly number[], + urlStr: string, +): Promise<{ shouldRetry: boolean; delayMs?: number }> { + // Check if this is a retryable error + const isCreditsExhausted = + response.status === 429 && (await isCreditsExhaustedError(response)) + + if ( + !isRetryableStatus( + response.status, + isCreditsExhausted, + retryableStatusCodes, + ) + ) { + // Not retryable - return the error response + return { shouldRetry: false } + } + + // If this is the last attempt, can't retry + if (attempt === maxRetries) { + logger.warn('HTTP request failed after max retries', { + url: urlStr, + statusCode: response.status, + attempts: attempt + 1, + maxRetries, + }) + return { shouldRetry: false } + } + + // Calculate delay + const retryAfterHeader = response.headers.get('Retry-After') + const retryAfterSeconds = parseRetryAfter(retryAfterHeader) + const delayMs = + retryAfterSeconds !== undefined + ? retryAfterSeconds * 1000 // Retry-After takes precedence + : calculateBackoffDelay(attempt, baseDelayMs, jitterMaxMs) + + logger.debug('HTTP request will be retried', { + url: urlStr, + statusCode: response.status, + attempt: attempt + 1, + nextRetryMs: delayMs, + retryAfterHeader, + }) + + return { shouldRetry: true, delayMs } +} + +type FetchWithPreconnect = FetchLike & { + preconnect?: typeof globalThis.fetch.preconnect +} + +function getBoundPreconnect( + fetchLike: FetchLike | typeof globalThis.fetch, +): typeof globalThis.fetch.preconnect | undefined { + const candidate = fetchLike as FetchWithPreconnect + if (typeof candidate.preconnect !== 'function') { + return undefined + } + + return candidate.preconnect.bind(candidate) +} + +/** + * Wraps a fetch function with exponential backoff retry logic. + * Respects Retry-After headers and includes jitter for thundering herd mitigation. + * + * @param fetch The fetch function to wrap + * @param options Retry configuration + * @returns A new fetch function with retry capability + */ +export function createRetryableFetch( + fetch: FetchLike, + options: RetryOptions = {}, +): typeof globalThis.fetch { + const maxRetries = options.maxRetries ?? HTTP_RETRY.MAX_RETRIES + const baseDelayMs = options.baseDelayMs ?? HTTP_RETRY.BASE_DELAY_MS + const jitterMaxMs = options.jitterMaxMs ?? HTTP_RETRY.JITTER_MAX_MS + const retryableStatusCodes = + options.retryableStatusCodes ?? HTTP_RETRY.RETRYABLE_STATUS_CODES + const retryableFetch = async ( + url: RequestInfo | URL, + options?: RequestInit, + ): Promise => { + const urlStr = typeof url === 'string' ? url : url.toString() + let lastError: Error | RetryableError | undefined + let lastResponse: Response | undefined + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + const response = await fetch(url, options) + + // If successful, return immediately + if (response.ok) { + return response + } + + // Evaluate if we should retry this response + const { shouldRetry, delayMs } = await evaluateResponseForRetry( + response, + attempt, + maxRetries, + baseDelayMs, + jitterMaxMs, + retryableStatusCodes, + urlStr, + ) + + if (!shouldRetry) { + return response + } + + // Store response for potential retry and wait before next attempt + lastResponse = response + if (delayMs === undefined) { + throw new Error('Expected retry delay to be defined') + } + await cancelResponseBody(response) + await wait(delayMs) + } catch (error) { + const backoffDelayMs = handleNetworkError( + error as Error, + attempt, + maxRetries, + baseDelayMs, + jitterMaxMs, + urlStr, + ) + lastError = error as Error + await wait(backoffDelayMs) + } + } + + // Exhausted retries - return last response or throw last error + if (lastResponse) { + return lastResponse + } + throw lastError ?? new Error('Unknown error during retry loop') + } + + const preconnect = + getBoundPreconnect(fetch) ?? getBoundPreconnect(globalThis.fetch) + + return Object.assign( + retryableFetch, + preconnect ? { preconnect } : {}, + ) as typeof globalThis.fetch +} + +/** + * Checks if a 429 response is due to exhausted credits. + * Reads and parses the response body, so be careful with streaming responses. + * + * @param response The HTTP response to check + * @returns Whether the error is CREDITS_EXHAUSTED + */ +async function isCreditsExhaustedError(response: Response): Promise { + try { + const cloned = response.clone() + const text = await cloned.text() + const parsed = JSON.parse(text) + return parsed.error?.code === 'CREDITS_EXHAUSTED' + } catch { + return false + } +} diff --git a/packages/browseros-agent/apps/server/tests/lib/browseros-fetch.test.ts b/packages/browseros-agent/apps/server/tests/lib/browseros-fetch.test.ts new file mode 100644 index 000000000..ec50b9bed --- /dev/null +++ b/packages/browseros-agent/apps/server/tests/lib/browseros-fetch.test.ts @@ -0,0 +1,192 @@ +/** + * @license + * Copyright 2025 BrowserOS + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * Integration tests for BrowserOS fetch with retry and rate-limit handling. + */ + +import { beforeEach, describe, expect, it, mock } from 'bun:test' +import { APICallError } from '@ai-sdk/provider' +import { createBrowserOSFetch } from '../../src/lib/browseros-fetch' + +describe('BrowserOS Fetch Integration', () => { + beforeEach(() => { + // Reset fetch mock before each test + globalThis.fetch = mock(async () => { + return new Response('OK', { status: 200 }) + }) + }) + + it('adds X-BrowserOS-ID header to requests', async () => { + const browserosId = 'test-123' + + let capturedHeaders: Headers | undefined + globalThis.fetch = mock(async (_url, options) => { + capturedHeaders = options?.headers as Headers + return new Response('OK', { status: 200 }) + }) + + const fetch = createBrowserOSFetch(browserosId) + await fetch('http://example.com') + + expect(capturedHeaders?.get('X-BrowserOS-ID')).toBe(browserosId) + }) + + it('throws APICallError with isRetryable=false for CREDITS_EXHAUSTED', async () => { + globalThis.fetch = mock(async () => { + return new Response( + JSON.stringify({ + error: { + code: 'CREDITS_EXHAUSTED', + message: 'Daily credits exhausted', + }, + }), + { + status: 429, + headers: { 'Content-Type': 'application/json' }, + }, + ) + }) + + const fetch = createBrowserOSFetch('test-123') + + try { + await fetch('http://example.com') + expect.unreachable() + } catch (error) { + expect(error instanceof APICallError).toBe(true) + const apiError = error as APICallError + expect(apiError.statusCode).toBe(429) + expect(apiError.isRetryable).toBe(false) + expect(apiError.message).toContain('Daily credits exhausted') + } + }) + + it('retries on 429 without CREDITS_EXHAUSTED code', async () => { + let attempts = 0 + globalThis.fetch = mock(async () => { + attempts++ + if (attempts < 2) { + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMIT_EXCEEDED', + message: 'Rate limit exceeded', + }, + }), + { + status: 429, + headers: { + 'Content-Type': 'application/json', + 'Retry-After': '0', + }, + }, + ) + } + return new Response('OK', { status: 200 }) + }) + + const fetch = createBrowserOSFetch('test-123') + const result = await fetch('http://example.com') + + expect(result.status).toBe(200) + expect(attempts).toBe(2) + }) + + it('extracts error details from response body', async () => { + globalThis.fetch = mock(async () => { + return new Response( + JSON.stringify({ + error: { + code: 'SOME_ERROR', + message: 'Something went wrong', + metadata: { + raw: { details: 'Additional context' }, + }, + }, + }), + { + status: 500, + headers: { + 'Content-Type': 'application/json', + 'Retry-After': '0', + }, + }, + ) + }) + + const fetch = createBrowserOSFetch('test-123') + + try { + await fetch('http://example.com') + expect.unreachable() + } catch (error) { + const apiError = error as APICallError + expect(apiError.message).toContain('SOME_ERROR') + expect(apiError.message).toContain('Something went wrong') + } + }) + + it('parses Retry-After header from 429 responses', async () => { + globalThis.fetch = mock(async () => { + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMIT', + message: 'Rate limited', + }, + }), + { + status: 429, + headers: { + 'Content-Type': 'application/json', + 'Retry-After': '0', + }, + }, + ) + }) + + const fetch = createBrowserOSFetch('test-123') + + try { + await fetch('http://example.com') + } catch (error) { + const apiError = error as APICallError + expect(apiError.responseHeaders?.['retry-after']).toBe('0') + } + }) + + it('throws APICallError for non-retryable errors like 404', async () => { + globalThis.fetch = mock(async () => { + return new Response('Not Found', { status: 404 }) + }) + + const fetch = createBrowserOSFetch('test-123') + + try { + await fetch('http://example.com') + expect.unreachable() + } catch (error) { + expect(error instanceof APICallError).toBe(true) + const apiError = error as APICallError + expect(apiError.statusCode).toBe(404) + expect(apiError.isRetryable).toBe(false) + } + }) + + it('logs credits remaining from response header', async () => { + globalThis.fetch = mock(async () => { + return new Response('OK', { + status: 200, + headers: { 'X-Credits-Remaining': '95' }, + }) + }) + + const fetch = createBrowserOSFetch('test-123') + const response = await fetch('http://example.com') + + expect(response.status).toBe(200) + expect(response.headers.get('X-Credits-Remaining')).toBe('95') + }) +}) diff --git a/packages/browseros-agent/apps/server/tests/lib/retry-handler.test.ts b/packages/browseros-agent/apps/server/tests/lib/retry-handler.test.ts new file mode 100644 index 000000000..0848f8841 --- /dev/null +++ b/packages/browseros-agent/apps/server/tests/lib/retry-handler.test.ts @@ -0,0 +1,324 @@ +/** + * @license + * Copyright 2025 BrowserOS + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * Tests for HTTP retry handler with exponential backoff and Retry-After support. + */ + +import { beforeEach, describe, expect, it, mock } from 'bun:test' +import { + calculateBackoffDelay, + createRetryableFetch, + isRetryableStatus, + parseRetryAfter, +} from '../../src/lib/retry-handler' + +describe('Retry Handler', () => { + describe('parseRetryAfter', () => { + it('parses numeric retry-after as seconds', () => { + expect(parseRetryAfter('60')).toBe(60) + expect(parseRetryAfter('120')).toBe(120) + }) + + it('parses HTTP-date retry-after', () => { + // Create a date 10 seconds in the future + const futureDate = new Date(Date.now() + 10_000) + const httpDate = futureDate.toUTCString() + const parsed = parseRetryAfter(httpDate) + + if (parsed === undefined) { + throw new Error('Expected Retry-After date to parse') + } + + expect(parsed).toBeGreaterThan(8) // Allow some variance + expect(parsed).toBeLessThan(12) + }) + + it('returns undefined for invalid retry-after', () => { + expect(parseRetryAfter('invalid')).toBeUndefined() + expect(parseRetryAfter('')).toBeUndefined() + expect(parseRetryAfter(null)).toBeUndefined() + expect(parseRetryAfter(undefined)).toBeUndefined() + }) + + it('returns undefined for past dates', () => { + const pastDate = new Date(Date.now() - 10_000) + const httpDate = pastDate.toUTCString() + expect(parseRetryAfter(httpDate)).toBeUndefined() + }) + }) + + describe('calculateBackoffDelay', () => { + it('calculates exponential backoff without exceeding max jitter', () => { + const baseDelay = 100 + const jitterMax = 50 + + for (let attempt = 0; attempt < 5; attempt++) { + const delay = calculateBackoffDelay(attempt, baseDelay, jitterMax) + const minDelay = baseDelay * 2 ** attempt + const maxDelay = minDelay + jitterMax + + expect(delay).toBeGreaterThanOrEqual(minDelay) + expect(delay).toBeLessThanOrEqual(maxDelay) + } + }) + + it('increases delay exponentially with attempt number', () => { + const baseDelay = 100 + const jitterMax = 0 // No jitter for consistent testing + + const delay0 = calculateBackoffDelay(0, baseDelay, jitterMax) + const delay1 = calculateBackoffDelay(1, baseDelay, jitterMax) + const delay2 = calculateBackoffDelay(2, baseDelay, jitterMax) + + expect(delay1).toBe(delay0 * 2) + expect(delay2).toBe(delay1 * 2) + }) + }) + + describe('isRetryableStatus', () => { + it('retries on 429 (rate limit)', () => { + expect(isRetryableStatus(429)).toBe(true) + }) + + it('does not retry on 429 with CREDITS_EXHAUSTED', () => { + expect(isRetryableStatus(429, true)).toBe(false) + }) + + it('retries on server errors (5xx)', () => { + expect(isRetryableStatus(500)).toBe(true) + expect(isRetryableStatus(502)).toBe(true) + expect(isRetryableStatus(503)).toBe(true) + expect(isRetryableStatus(504)).toBe(true) + }) + + it('retries on 408 (timeout)', () => { + expect(isRetryableStatus(408)).toBe(true) + }) + + it('does not retry on client errors (4xx except 429, 408)', () => { + expect(isRetryableStatus(400)).toBe(false) + expect(isRetryableStatus(401)).toBe(false) + expect(isRetryableStatus(403)).toBe(false) + expect(isRetryableStatus(404)).toBe(false) + }) + + it('supports custom retryable status codes', () => { + const customCodes = [418, 429, 500] as const + expect(isRetryableStatus(418, false, customCodes)).toBe(true) + expect(isRetryableStatus(403, false, customCodes)).toBe(false) + }) + }) + + describe('createRetryableFetch', () => { + let fetchMock: ReturnType + + beforeEach(() => { + fetchMock = mock(async () => { + throw new Error('Network error') + }) + }) + + it('returns successful response immediately', async () => { + const response = new Response('OK', { status: 200 }) + fetchMock = mock(async () => response) + + const retryableFetch = createRetryableFetch(fetchMock as typeof fetch) + const result = await retryableFetch('http://example.com') + + expect(result).toBe(response) + expect(fetchMock.mock.calls.length).toBe(1) + }) + + it('retries on network errors', async () => { + let attempts = 0 + fetchMock = mock(async () => { + attempts++ + if (attempts < 3) { + throw new Error('Network error') + } + return new Response('OK', { status: 200 }) + }) + + const retryableFetch = createRetryableFetch(fetchMock as typeof fetch, { + maxRetries: 3, + baseDelayMs: 1, // Minimal delay for testing + jitterMaxMs: 0, + }) + + const result = await retryableFetch('http://example.com') + expect(result.status).toBe(200) + expect(attempts).toBe(3) + }) + + it('respects max retries limit', async () => { + fetchMock = mock(async () => { + throw new Error('Network error') + }) + + const retryableFetch = createRetryableFetch(fetchMock as typeof fetch, { + maxRetries: 2, + baseDelayMs: 1, + jitterMaxMs: 0, + }) + + try { + await retryableFetch('http://example.com') + } catch (error) { + expect(error instanceof Error).toBe(true) + } + + // Original attempt + 2 retries = 3 total + expect(fetchMock.mock.calls.length).toBe(3) + }) + + it('retries on 429 rate limit errors', async () => { + let attempts = 0 + fetchMock = mock(async () => { + attempts++ + if (attempts < 2) { + return new Response('Too Many Requests', { + status: 429, + headers: { 'Content-Type': 'application/json' }, + }) + } + return new Response('OK', { status: 200 }) + }) + + const retryableFetch = createRetryableFetch(fetchMock as typeof fetch, { + maxRetries: 3, + baseDelayMs: 1, + jitterMaxMs: 0, + }) + + const result = await retryableFetch('http://example.com') + expect(result.status).toBe(200) + expect(attempts).toBe(2) + }) + + it('respects Retry-After header in seconds', async () => { + let attempts = 0 + const startTime = Date.now() + fetchMock = mock(async () => { + attempts++ + if (attempts < 2) { + return new Response('Too Many Requests', { + status: 429, + headers: { + 'Retry-After': '1', + 'Content-Type': 'application/json', + }, + }) + } + return new Response('OK', { status: 200 }) + }) + + const retryableFetch = createRetryableFetch(fetchMock as typeof fetch, { + maxRetries: 3, + baseDelayMs: 100, + jitterMaxMs: 0, + }) + + const result = await retryableFetch('http://example.com') + const elapsedMs = Date.now() - startTime + + expect(result.status).toBe(200) + expect(attempts).toBe(2) + // Should wait at least 1000ms due to Retry-After header + expect(elapsedMs).toBeGreaterThanOrEqual(900) // Allow 100ms variance + }) + + it('does not retry on non-retryable 4xx errors', async () => { + fetchMock = mock(async () => { + return new Response('Not Found', { + status: 404, + headers: { 'Content-Type': 'application/json' }, + }) + }) + + const retryableFetch = createRetryableFetch(fetchMock as typeof fetch, { + maxRetries: 3, + baseDelayMs: 1, + jitterMaxMs: 0, + }) + + const result = await retryableFetch('http://example.com') + expect(result.status).toBe(404) + expect(fetchMock.mock.calls.length).toBe(1) // Only one attempt + }) + + it('retries on 500+ server errors', async () => { + let attempts = 0 + fetchMock = mock(async () => { + attempts++ + if (attempts < 2) { + return new Response('Internal Server Error', { status: 500 }) + } + return new Response('OK', { status: 200 }) + }) + + const retryableFetch = createRetryableFetch(fetchMock as typeof fetch, { + maxRetries: 3, + baseDelayMs: 1, + jitterMaxMs: 0, + }) + + const result = await retryableFetch('http://example.com') + expect(result.status).toBe(200) + expect(attempts).toBe(2) + }) + + it('applies exponential backoff between retries', async () => { + let attempts = 0 + const attemptTimes: number[] = [] + fetchMock = mock(async () => { + attemptTimes.push(Date.now()) + attempts++ + if (attempts < 3) { + return new Response('Too Many Requests', { status: 429 }) + } + return new Response('OK', { status: 200 }) + }) + + const baseDelay = 50 + const retryableFetch = createRetryableFetch(fetchMock as typeof fetch, { + maxRetries: 3, + baseDelayMs: baseDelay, + jitterMaxMs: 0, + }) + + await retryableFetch('http://example.com') + + // Check that delays increase exponentially + const delay1 = attemptTimes[1] - attemptTimes[0] + const delay2 = attemptTimes[2] - attemptTimes[1] + + expect(delay1).toBeGreaterThanOrEqual(baseDelay) + expect(delay2).toBeGreaterThanOrEqual(baseDelay * 2) + }) + + it('handles custom retryable status codes', async () => { + let attempts = 0 + fetchMock = mock(async () => { + attempts++ + if (attempts < 2) { + return new Response('I am a teapot', { status: 418 }) + } + return new Response('OK', { status: 200 }) + }) + + const customRetryableCodes = [418, 429, 500, 502, 503, 504] as const + const retryableFetch = createRetryableFetch(fetchMock as typeof fetch, { + maxRetries: 3, + baseDelayMs: 1, + jitterMaxMs: 0, + retryableStatusCodes: customRetryableCodes, + }) + + const result = await retryableFetch('http://example.com') + expect(result.status).toBe(200) + expect(attempts).toBe(2) + }) + }) +}) diff --git a/packages/browseros-agent/packages/shared/src/constants/timeouts.ts b/packages/browseros-agent/packages/shared/src/constants/timeouts.ts index ca3c8882b..61830302e 100644 --- a/packages/browseros-agent/packages/shared/src/constants/timeouts.ts +++ b/packages/browseros-agent/packages/shared/src/constants/timeouts.ts @@ -10,6 +10,19 @@ export const KLAVIS_PROXY_RETRY_BACKOFF_MS = [ 5_000, 10_000, 20_000, 40_000, 60_000, ] as const +/** + * HTTP retry configuration for rate-limited and transient failures. + * Uses exponential backoff: baseDelay * 2^attempt + jitter + */ +export const HTTP_RETRY = { + // Exponential backoff multiplier: delay = baseDelay * 2^attempt + random(0, jitter) + BASE_DELAY_MS: 200, // Start with 200ms + JITTER_MAX_MS: 100, // Random 0-100ms added to each retry + MAX_RETRIES: 5, // Maximum of 5 retries (6 total attempts including original) + // Status codes to retry on + RETRYABLE_STATUS_CODES: [408, 429, 500, 502, 503, 504] as const, +} as const + export const TIMEOUTS = { // Agent/Tool execution TOOL_CALL: 120_000,