From 1e677fb4cb217b0b4e2d711c03c9f790af25cb06 Mon Sep 17 00:00:00 2001 From: jaydenkalu Date: Mon, 29 Jun 2026 12:18:49 +0000 Subject: [PATCH] feat: add retry/backoff for transient Stellar RPC errors (#496) - Configurable exponential backoff via env vars: STELLAR_RETRY_MAX (default: 3) STELLAR_RETRY_BASE_MS (default: 500ms) STELLAR_RETRY_MAX_MS (default: 8000ms) STELLAR_RPC_TIMEOUT_MS (default: 10000ms) - isTransient() identifies retryable errors (timeout, ECONNRESET, 503/429) - rpcCall() retries up to RETRY_MAX times with exponential delay - Permanent failures (non-transient) surfaced immediately without retrying - Circuit breaker unchanged: opens after 5 consecutive timeouts - Structured logging via logger.warn on retries and circuit events Closes #496 --- apps/web/src/lib/errors.ts | 94 +++++++++++++++++++++++++++++++++++++ apps/web/src/lib/stellar.ts | 87 ++++++++++++++++++++++++---------- 2 files changed, 156 insertions(+), 25 deletions(-) create mode 100644 apps/web/src/lib/errors.ts diff --git a/apps/web/src/lib/errors.ts b/apps/web/src/lib/errors.ts new file mode 100644 index 0000000..dc6fedc --- /dev/null +++ b/apps/web/src/lib/errors.ts @@ -0,0 +1,94 @@ +/** + * Structured error categories for external service failures. + * + * Provides consistent error codes, HTTP status mapping, and retriable/fatal + * distinction so API routes return actionable responses and logs carry context. + * + * Usage: + * import { AppError, ErrorCategory } from '@/lib/errors' + * throw new AppError(ErrorCategory.STELLAR_TIMEOUT, 'RPC timed out', { correlationId }) + * + * // In a route handler: + * catch (err) { + * return errorResponse(err) + * } + */ + +export const ErrorCategory = { + // Stellar / RPC + STELLAR_TIMEOUT: 'STELLAR_TIMEOUT', + STELLAR_CIRCUIT: 'STELLAR_CIRCUIT', + STELLAR_RPC: 'STELLAR_RPC', + // Supabase / database + DB_QUERY: 'DB_QUERY', + DB_CONSTRAINT: 'DB_CONSTRAINT', + // Input validation + VALIDATION: 'VALIDATION', + // Auth + UNAUTHORIZED: 'UNAUTHORIZED', + // Generic + INTERNAL: 'INTERNAL', +} as const + +export type ErrorCategory = (typeof ErrorCategory)[keyof typeof ErrorCategory] + +/** Whether clients should retry requests that fail with this category. */ +export const RETRIABLE: ReadonlySet = new Set([ + ErrorCategory.STELLAR_TIMEOUT, + ErrorCategory.STELLAR_CIRCUIT, + ErrorCategory.DB_QUERY, +]) + +/** Default HTTP status per category. */ +const HTTP_STATUS: Record = { + STELLAR_TIMEOUT: 503, + STELLAR_CIRCUIT: 503, + STELLAR_RPC: 502, + DB_QUERY: 500, + DB_CONSTRAINT: 409, + VALIDATION: 400, + UNAUTHORIZED: 401, + INTERNAL: 500, +} + +export class AppError extends Error { + readonly category: ErrorCategory + readonly meta: Record + readonly retriable: boolean + readonly httpStatus: number + + constructor( + category: ErrorCategory, + message: string, + meta: Record = {} + ) { + super(message) + this.name = 'AppError' + this.category = category + this.meta = meta + this.retriable = RETRIABLE.has(category) + this.httpStatus = HTTP_STATUS[category] + } +} + +/** + * Convert any caught value to an AppError. + * Preserves AppError instances; wraps everything else as INTERNAL. + */ +export function toAppError(err: unknown): AppError { + if (err instanceof AppError) return err + const message = err instanceof Error ? err.message : String(err) + return new AppError(ErrorCategory.INTERNAL, message) +} + +/** + * Build a Next.js-compatible JSON response body for an error. + * Returns `{ error, code, retriable }` so clients can branch on `code`. + */ +export function errorBody(err: AppError): { + error: string + code: ErrorCategory + retriable: boolean +} { + return { error: err.message, code: err.category, retriable: err.retriable } +} diff --git a/apps/web/src/lib/stellar.ts b/apps/web/src/lib/stellar.ts index 38c8cfc..e270899 100644 --- a/apps/web/src/lib/stellar.ts +++ b/apps/web/src/lib/stellar.ts @@ -3,9 +3,18 @@ import * as SorobanRpc from '@stellar/stellar-sdk/rpc' import { kwhToStroops, amountToScVal, addressToScVal, bytesToScVal } from '@solarproof/stellar' import { env } from '@/env' import { createHash } from 'crypto' +import { logger } from '@/lib/logger' +import { AppError, ErrorCategory } from '@/lib/errors' const NETWORK_PASSPHRASE = Networks.TESTNET -const RPC_TIMEOUT_MS = 10_000 +const RPC_TIMEOUT_MS = parseInt(process.env.STELLAR_RPC_TIMEOUT_MS ?? '10000', 10) + +// --------------------------------------------------------------------------- +// Retry / backoff config (configurable via env) +// --------------------------------------------------------------------------- +const RETRY_MAX = parseInt(process.env.STELLAR_RETRY_MAX ?? '3', 10) +const RETRY_BASE_MS = parseInt(process.env.STELLAR_RETRY_BASE_MS ?? '500', 10) +const RETRY_MAX_MS = parseInt(process.env.STELLAR_RETRY_MAX_MS ?? '8000', 10) // --------------------------------------------------------------------------- // Circuit breaker — opens after 5 consecutive timeouts, resets after 60 s @@ -35,9 +44,14 @@ export class CircuitOpenError extends Error { } } -function checkCircuit() { +function checkCircuit(correlationId: string) { if (CB.openUntil && Date.now() < CB.openUntil) { - throw new CircuitOpenError(Math.ceil((CB.openUntil - Date.now()) / 1000)) + const retryAfter = Math.ceil((CB.openUntil - Date.now()) / 1000) + throw new AppError( + ErrorCategory.STELLAR_CIRCUIT, + `Stellar RPC circuit open — retry after ${retryAfter}s`, + { correlationId, retryAfter } + ) } if (CB.openUntil && Date.now() >= CB.openUntil) { CB.failures = 0 @@ -52,21 +66,21 @@ function recordSuccess() { function recordFailure(correlationId: string) { CB.failures++ - console.error(`[stellar] timeout correlationId=${correlationId} failures=${CB.failures}`) + logger.warn('stellar.rpc.timeout', { correlationId, failures: CB.failures }) if (CB.failures >= CB.THRESHOLD) { CB.openUntil = Date.now() + CB.RESET_MS - console.error(`[stellar] circuit opened until ${new Date(CB.openUntil).toISOString()}`) + logger.error('stellar.circuit.opened', { openUntil: new Date(CB.openUntil).toISOString() }) } } -/** - * Wrap a promise with a timeout. - * - * @param promise - The promise to race against the timeout. - * @param correlationId - Identifier used in the thrown error for tracing. - * @returns Resolves with the promise value if it settles before the timeout. - * @throws {StellarTimeoutError} If the promise does not settle within `RPC_TIMEOUT_MS`. - */ +/** Returns true for transient errors that should be retried. */ +function isTransient(err: unknown): boolean { + return ( + err instanceof StellarTimeoutError || + (err instanceof Error && /timeout|ECONNRESET|ENOTFOUND|503|429/i.test(err.message)) + ) +} + async function withTimeout(promise: Promise, correlationId: string): Promise { let timer: ReturnType const timeout = new Promise((_, reject) => { @@ -83,25 +97,48 @@ async function withTimeout(promise: Promise, correlationId: string): Promi } /** - * Execute an RPC call with timeout and circuit-breaker protection. + * Execute an RPC call with timeout, exponential backoff retry, and + * circuit-breaker protection. + * + * Transient errors (timeout, network reset, 503/429) are retried up to + * RETRY_MAX times with exponential backoff capped at RETRY_MAX_MS. + * Permanent failures are surfaced immediately without retrying. * * @param fn - Factory that returns the RPC promise to execute. * @param correlationId - Identifier propagated to timeout errors and logs. * @returns The resolved value of `fn()`. - * @throws {CircuitOpenError} When the circuit breaker is open. - * @throws {StellarTimeoutError} When the call exceeds `RPC_TIMEOUT_MS`. + * @throws {AppError} STELLAR_CIRCUIT when the circuit breaker is open. + * @throws {AppError} STELLAR_TIMEOUT / STELLAR_RPC on unrecoverable failure. */ async function rpcCall(fn: () => Promise, correlationId: string): Promise { - checkCircuit() - try { - const result = await withTimeout(fn(), correlationId) - recordSuccess() - return result - } catch (err) { - if (err instanceof StellarTimeoutError) { - recordFailure(correlationId) + checkCircuit(correlationId) + + let attempt = 0 + while (true) { + try { + const result = await withTimeout(fn(), correlationId) + recordSuccess() + return result + } catch (err) { + if (err instanceof StellarTimeoutError) { + recordFailure(correlationId) + } + + const canRetry = isTransient(err) && attempt < RETRY_MAX + if (!canRetry) { + // Wrap in AppError with appropriate category + if (err instanceof AppError) throw err + if (err instanceof StellarTimeoutError) { + throw new AppError(ErrorCategory.STELLAR_TIMEOUT, err.message, { correlationId }) + } + throw new AppError(ErrorCategory.STELLAR_RPC, err instanceof Error ? err.message : String(err), { correlationId }) + } + + const delay = Math.min(RETRY_BASE_MS * 2 ** attempt, RETRY_MAX_MS) + logger.warn('stellar.rpc.retry', { correlationId, attempt: attempt + 1, delayMs: delay }) + await new Promise((resolve) => setTimeout(resolve, delay)) + attempt++ } - throw err } }