diff --git a/src/llm/openai.test.ts b/src/llm/openai.test.ts index 9b29864..e884fb3 100644 --- a/src/llm/openai.test.ts +++ b/src/llm/openai.test.ts @@ -97,7 +97,38 @@ describe("OpenAIClient.complete", () => { const client = makeClient(); stub(client, async () => ({ choices: [] })); await assert.rejects(client.complete({ user: "go", maxTokens: 5 }), { - message: /returned no content/, + message: /returned no choices/, }); }); + + it("throws when choices is undefined (e.g. OpenRouter error payload)", async () => { + const client = makeClient(); + stub(client, async () => ({ + error: { message: "The operation was aborted", code: 504 }, + })); + await assert.rejects(client.complete({ user: "go", maxTokens: 5 }), { + message: /returned no choices or an error payload/, + }); + }); + + it("retries on retryable error", async () => { + const client = makeClient(); + let callCount = 0; + stub(client, async () => { + callCount++; + if (callCount === 1) { + const err = new Error("upstream error") as Error & { + status: number; + }; + err.status = 502; + throw err; + } + return { + choices: [{ message: { content: "success" }, finish_reason: "stop" }], + }; + }); + const result = await client.complete({ user: "go", maxTokens: 5 }); + assert.equal(result, "success"); + assert.equal(callCount, 2); + }); }); diff --git a/src/llm/openai.ts b/src/llm/openai.ts index 5d73855..92f7831 100644 --- a/src/llm/openai.ts +++ b/src/llm/openai.ts @@ -4,13 +4,37 @@ import type { ChatCompletionMessageParam, } from "openai/resources/chat/completions"; import type { LLMClient, LLMCompleteParams } from "./types.js"; +import pc from "picocolors"; + +/** HTTP status codes worth retrying due to transient provider/network failures. */ +const RETRYABLE_CODES = new Set([429, 500, 502, 503, 504]); + +function isRetryableError(error: unknown): boolean { + if (error instanceof Error) { + const msg = error.message.toLowerCase(); + if ( + msg.includes("rate limit") || + msg.includes("timeout") || + msg.includes("econnrefused") || + msg.includes("econnreset") || + msg.includes("aborted") + ) + return true; + } + // OpenAI SDK attaches `status` on API errors. + const status = (error as { status?: number })?.status; + if (status && RETRYABLE_CODES.has(status)) return true; + return false; +} /** * OpenAI-compatible backend. By configuring `baseURL` this also drives Gemini * (generativelanguage…/openai/), Ollama (localhost:11434/v1), Groq, Together, - * and any other Chat Completions-compatible endpoint. No prompt caching — - * `response_format: json_object` is requested when supported, and the caller's - * JSON-repair path still runs as a safety net for endpoints that ignore it. + * NVIDIA NIM, OpenRouter, Mistral, and any other Chat Completions-compatible + * endpoint. + * + * The client retries retryable failures (429/5xx, empty choices, etc.) up to a + * small fixed number of attempts before propagating the error. */ export class OpenAIClient implements LLMClient { readonly label: string; @@ -27,6 +51,35 @@ export class OpenAIClient implements LLMClient { } async complete(params: LLMCompleteParams): Promise { + const MAX_RETRIES = 2; + let lastError: unknown; + + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + return await this.tryComplete(params); + } catch (error) { + lastError = error; + if (!isRetryableError(error) || attempt === MAX_RETRIES) break; + + if (process.stderr.isTTY) { + process.stderr.write( + pc.yellow( + ` ⚠ Request failed (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying...\n`, + ), + ); + } + + // Wait a bit before retrying 429s + if ((error as { status?: number })?.status === 429) { + await new Promise((r) => setTimeout(r, 2000)); + } + } + } + + throw lastError; + } + + private async tryComplete(params: LLMCompleteParams): Promise { const messages: ChatCompletionMessageParam[] = []; if (params.system) messages.push({ role: "system", content: params.system }); @@ -44,7 +97,22 @@ export class OpenAIClient implements LLMClient { resp = await this.client.chat.completions.create(request); } catch (error) { const message = error instanceof Error ? error.message : String(error); - throw new Error(`${this.label} request failed: ${message}`); + throw Object.assign( + new Error(`${this.label} request failed: ${message}`), + { status: (error as { status?: number })?.status }, + ); + } + + if (!resp.choices || resp.choices.length === 0) { + const respStr = JSON.stringify(resp); + const details = + respStr.length > 1000 ? `${respStr.slice(0, 1000)}…` : respStr; + throw Object.assign( + new Error( + `${this.label} returned no choices or an error payload: ${details}`, + ), + { status: 502 }, + ); } const choice = resp.choices[0];