From 74033e49714df4edceabb91ff9bd244205abae6c Mon Sep 17 00:00:00 2001 From: Tejas Date: Fri, 5 Jun 2026 20:02:57 +0530 Subject: [PATCH 1/4] fix(llm): retry on transient API errors (429, 50x) instead of crashing --- src/llm/openai.test.ts | 35 +++++++++++++++++++- src/llm/openai.ts | 73 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/src/llm/openai.test.ts b/src/llm/openai.test.ts index 9b29864..5df3600 100644 --- a/src/llm/openai.test.ts +++ b/src/llm/openai.test.ts @@ -97,7 +97,40 @@ describe("OpenAIClient.complete", () => { const client = makeClient(); stub(client, async () => ({ choices: [] })); await assert.rejects(client.complete({ user: "go", maxTokens: 5 }), { - message: /returned no content/, + message: /returned no choices/, }); }); + + it("throws when choices is undefined (e.g. OpenRouter error payload)", async () => { + const client = makeClient(); + stub(client, async () => ({ + error: { message: "The operation was aborted", code: 504 }, + })); + await assert.rejects(client.complete({ user: "go", maxTokens: 5 }), { + message: /returned no choices or an error payload/, + }); + }); + + it("retries on retryable error", async () => { + const client = makeClient(); + let callCount = 0; + stub(client, async () => { + callCount++; + if (callCount === 1) { + const err = new Error("rate limit exceeded") as Error & { + status: number; + }; + err.status = 429; + throw err; + } + return { + choices: [ + { message: { content: "success" }, finish_reason: "stop" }, + ], + }; + }); + const result = await client.complete({ user: "go", maxTokens: 5 }); + assert.equal(result, "success"); + assert.equal(callCount, 2); + }); }); diff --git a/src/llm/openai.ts b/src/llm/openai.ts index 5d73855..47cd39e 100644 --- a/src/llm/openai.ts +++ b/src/llm/openai.ts @@ -4,13 +4,38 @@ import type { ChatCompletionMessageParam, } from "openai/resources/chat/completions"; import type { LLMClient, LLMCompleteParams } from "./types.js"; +import pc from "picocolors"; + +/** HTTP status codes worth retrying with a different model. */ +const RETRYABLE_CODES = new Set([429, 500, 502, 503, 504]); + +function isRetryableError(error: unknown): boolean { + if (error instanceof Error) { + const msg = error.message.toLowerCase(); + if ( + msg.includes("rate limit") || + msg.includes("timeout") || + msg.includes("econnrefused") || + msg.includes("econnreset") || + msg.includes("aborted") + ) + return true; + } + // OpenAI SDK attaches `status` on API errors. + const status = (error as { status?: number })?.status; + if (status && RETRYABLE_CODES.has(status)) return true; + return false; +} /** * OpenAI-compatible backend. By configuring `baseURL` this also drives Gemini * (generativelanguage…/openai/), Ollama (localhost:11434/v1), Groq, Together, - * and any other Chat Completions-compatible endpoint. No prompt caching — - * `response_format: json_object` is requested when supported, and the caller's - * JSON-repair path still runs as a safety net for endpoints that ignore it. + * NVIDIA NIM, OpenRouter, Mistral, and any other Chat Completions-compatible + * endpoint. + * + * When `fallbackModels` are configured, a retryable failure (504, 429, empty + * choices, etc.) on the primary model triggers an automatic retry with the + * next fallback before the error propagates. */ export class OpenAIClient implements LLMClient { readonly label: string; @@ -27,6 +52,33 @@ export class OpenAIClient implements LLMClient { } async complete(params: LLMCompleteParams): Promise { + const MAX_RETRIES = 2; + let lastError: unknown; + + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + return await this.tryComplete(params); + } catch (error) { + lastError = error; + if (isRetryableError(error) && attempt < MAX_RETRIES) { + process.stderr.write( + pc.yellow( + ` ⚠ Request failed (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying...\n`, + ), + ); + // Wait a bit before retrying 429s + if ((error as { status?: number })?.status === 429) { + await new Promise((r) => setTimeout(r, 2000)); + } + continue; + } + throw error; + } + } + throw lastError; + } + + private async tryComplete(params: LLMCompleteParams): Promise { const messages: ChatCompletionMessageParam[] = []; if (params.system) messages.push({ role: "system", content: params.system }); @@ -44,7 +96,20 @@ export class OpenAIClient implements LLMClient { resp = await this.client.chat.completions.create(request); } catch (error) { const message = error instanceof Error ? error.message : String(error); - throw new Error(`${this.label} request failed: ${message}`); + throw Object.assign( + new Error(`${this.label} request failed: ${message}`), + { status: (error as { status?: number })?.status }, + ); + } + + if (!resp.choices || resp.choices.length === 0) { + const respStr = JSON.stringify(resp); + throw Object.assign( + new Error( + `${this.label} returned no choices or an error payload: ${respStr}`, + ), + { status: 502 }, + ); } const choice = resp.choices[0]; From b8f9daab9c4f3dda27104bc995170b33d761a3ba Mon Sep 17 00:00:00 2001 From: Tejas Date: Fri, 5 Jun 2026 22:20:55 +0530 Subject: [PATCH 2/4] fix: apply code review suggestions for resilience and documentation --- src/llm/openai.test.ts | 4 ++-- src/llm/openai.ts | 24 ++++++++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/llm/openai.test.ts b/src/llm/openai.test.ts index 5df3600..5168bee 100644 --- a/src/llm/openai.test.ts +++ b/src/llm/openai.test.ts @@ -117,10 +117,10 @@ describe("OpenAIClient.complete", () => { stub(client, async () => { callCount++; if (callCount === 1) { - const err = new Error("rate limit exceeded") as Error & { + const err = new Error("upstream error") as Error & { status: number; }; - err.status = 429; + err.status = 502; throw err; } return { diff --git a/src/llm/openai.ts b/src/llm/openai.ts index 47cd39e..c9eea2a 100644 --- a/src/llm/openai.ts +++ b/src/llm/openai.ts @@ -6,7 +6,7 @@ import type { import type { LLMClient, LLMCompleteParams } from "./types.js"; import pc from "picocolors"; -/** HTTP status codes worth retrying with a different model. */ +/** HTTP status codes worth retrying due to transient provider/network failures. */ const RETRYABLE_CODES = new Set([429, 500, 502, 503, 504]); function isRetryableError(error: unknown): boolean { @@ -33,9 +33,8 @@ function isRetryableError(error: unknown): boolean { * NVIDIA NIM, OpenRouter, Mistral, and any other Chat Completions-compatible * endpoint. * - * When `fallbackModels` are configured, a retryable failure (504, 429, empty - * choices, etc.) on the primary model triggers an automatic retry with the - * next fallback before the error propagates. + * The client retries retryable failures (429/5xx, empty choices, etc.) up to a + * small fixed number of attempts before propagating the error. */ export class OpenAIClient implements LLMClient { readonly label: string; @@ -61,11 +60,13 @@ export class OpenAIClient implements LLMClient { } catch (error) { lastError = error; if (isRetryableError(error) && attempt < MAX_RETRIES) { - process.stderr.write( - pc.yellow( - ` ⚠ Request failed (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying...\n`, - ), - ); + if (process.stderr.isTTY) { + process.stderr.write( + pc.yellow( + ` ⚠ Request failed (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying...\n`, + ), + ); + } // Wait a bit before retrying 429s if ((error as { status?: number })?.status === 429) { await new Promise((r) => setTimeout(r, 2000)); @@ -104,14 +105,17 @@ export class OpenAIClient implements LLMClient { if (!resp.choices || resp.choices.length === 0) { const respStr = JSON.stringify(resp); + const details = + respStr.length > 1000 ? `${respStr.slice(0, 1000)}…` : respStr; throw Object.assign( new Error( - `${this.label} returned no choices or an error payload: ${respStr}`, + `${this.label} returned no choices or an error payload: ${details}`, ), { status: 502 }, ); } + const choice = resp.choices[0]; const content = choice?.message?.content; if (!content) { From 9c24551a3d39663f62feab4df4d331d877cd3510 Mon Sep 17 00:00:00 2001 From: Tejas Date: Fri, 5 Jun 2026 22:31:34 +0530 Subject: [PATCH 3/4] fix: resolve prettier formatting violations --- src/llm/openai.test.ts | 4 +--- src/llm/openai.ts | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/llm/openai.test.ts b/src/llm/openai.test.ts index 5168bee..e884fb3 100644 --- a/src/llm/openai.test.ts +++ b/src/llm/openai.test.ts @@ -124,9 +124,7 @@ describe("OpenAIClient.complete", () => { throw err; } return { - choices: [ - { message: { content: "success" }, finish_reason: "stop" }, - ], + choices: [{ message: { content: "success" }, finish_reason: "stop" }], }; }); const result = await client.complete({ user: "go", maxTokens: 5 }); diff --git a/src/llm/openai.ts b/src/llm/openai.ts index c9eea2a..9a8d83d 100644 --- a/src/llm/openai.ts +++ b/src/llm/openai.ts @@ -115,7 +115,6 @@ export class OpenAIClient implements LLMClient { ); } - const choice = resp.choices[0]; const content = choice?.message?.content; if (!content) { From a24aa01fa229aae6d37467d9257ea386a36befc9 Mon Sep 17 00:00:00 2001 From: Tejas Date: Tue, 9 Jun 2026 11:22:33 +0530 Subject: [PATCH 4/4] fix: clean up loop structure for resilience and verify braces --- src/llm/openai.ts | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/llm/openai.ts b/src/llm/openai.ts index 9a8d83d..92f7831 100644 --- a/src/llm/openai.ts +++ b/src/llm/openai.ts @@ -59,23 +59,23 @@ export class OpenAIClient implements LLMClient { return await this.tryComplete(params); } catch (error) { lastError = error; - if (isRetryableError(error) && attempt < MAX_RETRIES) { - if (process.stderr.isTTY) { - process.stderr.write( - pc.yellow( - ` ⚠ Request failed (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying...\n`, - ), - ); - } - // Wait a bit before retrying 429s - if ((error as { status?: number })?.status === 429) { - await new Promise((r) => setTimeout(r, 2000)); - } - continue; + if (!isRetryableError(error) || attempt === MAX_RETRIES) break; + + if (process.stderr.isTTY) { + process.stderr.write( + pc.yellow( + ` ⚠ Request failed (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying...\n`, + ), + ); + } + + // Wait a bit before retrying 429s + if ((error as { status?: number })?.status === 429) { + await new Promise((r) => setTimeout(r, 2000)); } - throw error; } } + throw lastError; }