From 62cbab25bbdb14e0d61e019812c864c6e180271f Mon Sep 17 00:00:00 2001 From: Diwakar Singh Maurya Date: Sat, 29 Aug 2026 22:05:08 +0530 Subject: [PATCH 1/4] feat(openai): support structured output via response_format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OpenAI endpoint dropped `response_format` entirely. Requests asking for schema-valid JSON got prose back with no error Translate `response_format` into the `output_config.format` that the internal /v1/messages hop already understands. Enforcement is the SDK's native structured output, including schema validation and retry, so this is plumbing rather than a second implementation. Streaming works too: the SDK buffers until the result validates, then emits it as one delta. - `json_object` has no Anthropic equivalent — there is no schema-less JSON mode — so it is rejected rather than widened into a permissive object schema that would promise an enforcement the request never gets. - `strict` and `name` are dropped: the SDK always validates, which is never weaker than strict asked for, and `name` is a client-side label. Tools remain incompatible with structured output, unchanged. Errors name the field the caller actually sent. parseOutputFormat takes the paths to report, and the OpenAI route validates before the hop — otherwise a client sending `response_format` is told about `output_config.format`, a field it never wrote. Also strip a root-level `$schema` before handing the schema to the SDK. Anything but the draft-07 URI made the model fail to submit its result, returning HTTP 500 ("no structured_output result") after burning the turn budget — and zod v4's `z.toJSONSchema()` emits the 2020-12 URI by default, so every schema from that toolchain hit it. This affected the native /v1/messages endpoint equally. The keyword only declares a dialect and constrains nothing, so it is dropped rather than translated. --- src/__tests__/openai.test.ts | 62 +++++++++++++++++++ src/__tests__/proxy-openai-compat.test.ts | 56 +++++++++++++++++ src/__tests__/structured-output-parse.test.ts | 62 +++++++++++++++++++ src/proxy/openai.ts | 47 +++++++++++++- src/proxy/server.ts | 13 ++++ src/proxy/structuredOutput.ts | 57 +++++++++++++++-- 6 files changed, 288 insertions(+), 9 deletions(-) create mode 100644 src/__tests__/structured-output-parse.test.ts diff --git a/src/__tests__/openai.test.ts b/src/__tests__/openai.test.ts index f431451d..198b88a0 100644 --- a/src/__tests__/openai.test.ts +++ b/src/__tests__/openai.test.ts @@ -732,6 +732,68 @@ describe("translateOpenAiToAnthropic", () => { expect(result!.tools).toHaveLength(1) expect(result!.tools![0]!.description).toBe("") }) + + // --- structured output --- + + const personSchema = { + type: "object", + properties: { name: { type: "string" } }, + required: ["name"], + } + + it("maps response_format json_schema to output_config.format", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: { + type: "json_schema", + json_schema: { name: "person", schema: personSchema, strict: true }, + }, + }) + // `name` and `strict` have no Anthropic equivalent and are dropped: the + // SDK always validates, which is never weaker than strict asked for. + expect(result!.output_config).toEqual({ + format: { type: "json_schema", schema: personSchema }, + }) + }) + + it("keeps response_format alongside output_config.effort", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + output_config: { effort: "high" }, + response_format: { type: "json_schema", json_schema: { schema: personSchema } }, + }) + expect(result!.output_config).toEqual({ + effort: "high", + format: { type: "json_schema", schema: personSchema }, + }) + }) + + it("passes through an Anthropic-shaped output_config.format unchanged", () => { + const format = { type: "json_schema", schema: personSchema } + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + output_config: { format }, + }) + expect(result!.output_config).toEqual({ format }) + }) + + it("forwards json_object intact so the boundary can reject it", () => { + // Not widened into a permissive schema: Anthropic has no schema-less JSON + // mode, and accepting it here would promise an enforcement never applied. + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: { type: "json_object" }, + }) + expect(result!.output_config).toEqual({ format: { type: "json_object" } }) + }) + + it("treats response_format text as no constraint", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: { type: "text" }, + }) + expect(result!.output_config).toBeUndefined() + }) }) // --------------------------------------------------------------------------- diff --git a/src/__tests__/proxy-openai-compat.test.ts b/src/__tests__/proxy-openai-compat.test.ts index 64c65958..74ea097b 100644 --- a/src/__tests__/proxy-openai-compat.test.ts +++ b/src/__tests__/proxy-openai-compat.test.ts @@ -198,6 +198,62 @@ describe("POST /v1/chat/completions — non-streaming", () => { expect(capturedOptions?.effort).toBe("high") }) + it("carries response_format json_schema through to the SDK outputFormat", async () => { + // Structured output is enforced by the SDK on the internal /v1/messages + // hop. Without this the field is dropped at the endpoint boundary and the + // client silently gets prose back instead of schema-valid JSON. + const schema = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + additionalProperties: false, + } + mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] + const app = createTestApp() + + await postChatCompletion(app, { + stream: false, + response_format: { type: "json_schema", json_schema: { name: "answer", schema } }, + messages: [{ role: "user", content: "Hi" }], + }) + + expect(capturedOptions?.outputFormat).toEqual({ type: "json_schema", schema }) + }) + + it("rejects response_format json_object instead of silently ignoring it", async () => { + // Anthropic has no schema-less JSON mode, so the request cannot be honored. + // Failing loudly beats returning prose to a client expecting JSON. + mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: false, + response_format: { type: "json_object" }, + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(400) + }) + + it("rejects response_format combined with tools", async () => { + // Structured-output mode replaces the response content, which would swallow + // a tool_use turn and strand the client's tool loop. + mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: false, + response_format: { + type: "json_schema", + json_schema: { schema: { type: "object", properties: {} } }, + }, + tools: [{ type: "function", function: { name: "fn", parameters: {} } }], + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(400) + }) + it("sends the client system prompt verbatim, without the claude_code preset", async () => { // The OpenAI endpoint serves generic chat clients (Open WebUI, curl). // Their system prompt must reach the SDK as a plain string — NOT wrapped diff --git a/src/__tests__/structured-output-parse.test.ts b/src/__tests__/structured-output-parse.test.ts new file mode 100644 index 00000000..014b357b --- /dev/null +++ b/src/__tests__/structured-output-parse.test.ts @@ -0,0 +1,62 @@ +/** + * Unit tests for src/proxy/structuredOutput.ts - pure parsing/normalization. + * No I/O, no mocks required. + */ + +import { describe, it, expect } from "bun:test" +import { parseOutputFormat } from "../proxy/structuredOutput" + +const schema = { + type: "object", + additionalProperties: false, + required: ["a"], + properties: { a: { type: "integer" } }, +} + +function parseSchema(input: Record) { + const result = parseOutputFormat({ format: { type: "json_schema", schema: input } }) + if (!result.ok) throw new Error(`expected ok, got: ${result.message}`) + return result.value as { type: "json_schema"; schema: Record } +} + +describe("parseOutputFormat - $schema handling", () => { + it("strips a root-level $schema before reaching the SDK", () => { + // Any dialect but draft-07 makes the model fail to submit its structured + // result, which surfaces as a 500. The keyword constrains nothing, so it + // is dropped rather than passed through. + const value = parseSchema({ + $schema: "https://json-schema.org/draft/2020-12/schema", + ...schema, + }) + expect(value.schema).toEqual(schema) + expect("$schema" in value.schema).toBe(false) + }) + + it("strips the draft-07 $schema too, for one consistent shape", () => { + const value = parseSchema({ $schema: "http://json-schema.org/draft-07/schema#", ...schema }) + expect(value.schema).toEqual(schema) + }) + + it("leaves a schema without $schema untouched", () => { + const value = parseSchema({ ...schema }) + expect(value.schema).toEqual(schema) + }) + + it("does not mutate the caller's schema object", () => { + const input = { $schema: "https://json-schema.org/draft/2020-12/schema", ...schema } + parseSchema(input) + expect(input.$schema).toBe("https://json-schema.org/draft/2020-12/schema") + }) + + it("keeps a nested $schema, which the SDK already tolerates", () => { + // Only the root keyword breaks submission. Rewriting deeper would risk + // clobbering a property legitimately named `$schema`. + const nested = { + type: "object", + properties: { + p: { $schema: "https://json-schema.org/draft/2020-12/schema", type: "object" }, + }, + } + expect(parseSchema(nested).schema).toEqual(nested) + }) +}) diff --git a/src/proxy/openai.ts b/src/proxy/openai.ts index be6bdff3..71bc38c9 100644 --- a/src/proxy/openai.ts +++ b/src/proxy/openai.ts @@ -67,6 +67,15 @@ export interface OpenAiChatToolCustom { export type OpenAiChatTool = OpenAiChatToolFunction | OpenAiChatToolCustom +export interface OpenAiResponseFormat { + type: "text" | "json_object" | "json_schema" + json_schema?: { + name?: string + schema?: unknown + strict?: boolean + } +} + export interface OpenAiChatRequest { model?: string messages?: OpenAiMessage[] @@ -79,7 +88,9 @@ export interface OpenAiChatRequest { /** Standard OpenAI reasoning level (low/medium/high/…). */ reasoning_effort?: string /** Anthropic-style nesting some clients use. */ - output_config?: { effort?: string } + output_config?: { effort?: string; format?: unknown } + /** Standard OpenAI structured output (json_schema / json_object / text). */ + response_format?: OpenAiResponseFormat stream_options?: { include_usage?: boolean } } @@ -128,7 +139,7 @@ export interface AnthropicRequestBody { /** Reasoning effort carried from the OpenAI request so the internal * /v1/messages hop forwards it to the SDK (value gated by normalizeEffort). */ reasoning_effort?: string - output_config?: { effort?: string } + output_config?: { effort?: string; format?: unknown } } export interface AnthropicUsage { @@ -473,6 +484,24 @@ function summarizeAnthropicContent(content: string | AnthropicContentBlock[]): s * * Returns null if the request has no messages (caller should return 400). */ +/** + * Map OpenAI's `response_format` onto Anthropic's `output_config.format`. + * + * `json_object` is forwarded intact rather than widened into a permissive + * `{"type":"object"}` schema: Anthropic has no schema-less JSON mode, and + * accepting it silently would promise an enforcement the request never gets. + * parseOutputFormat rejects it with an actionable message. + */ +function translateResponseFormat(format: OpenAiResponseFormat | undefined): unknown { + if (format === undefined || format.type === "text") return undefined + // `name` is a client-side label; `strict` has no equivalent - the SDK always + // validates, which is never weaker than strict asked for. + if (format.type === "json_schema") { + return { type: "json_schema", schema: format.json_schema?.schema } + } + return { type: format.type } +} + export function translateOpenAiToAnthropic( body: OpenAiChatRequest, options: OpenAiTranslationOptions = {}, @@ -622,7 +651,19 @@ export function translateOpenAiToAnthropic( // and OpenAI clients always run at the model default. Validation happens // downstream via normalizeEffort. if (body.reasoning_effort !== undefined) result.reasoning_effort = body.reasoning_effort - if (body.output_config?.effort !== undefined) result.output_config = { effort: body.output_config.effort } + + // Structured output. `response_format` is the standard OpenAI spelling; + // `output_config.format` is accepted too because some clients send the + // Anthropic shape at this endpoint. Both are forwarded unvalidated so the + // single check in parseOutputFormat rejects them at the HTTP boundary. + const outputFormat = translateResponseFormat(body.response_format) ?? body.output_config?.format + const effort = body.output_config?.effort + if (effort !== undefined || outputFormat !== undefined) { + const outputConfig: NonNullable = {} + if (effort !== undefined) outputConfig.effort = effort + if (outputFormat !== undefined) outputConfig.format = outputFormat + result.output_config = outputConfig + } return result } diff --git a/src/proxy/server.ts b/src/proxy/server.ts index 1d8177f2..bfbea149 100644 --- a/src/proxy/server.ts +++ b/src/proxy/server.ts @@ -7014,6 +7014,19 @@ export function createProxyServer(config: Partial = {}): ProxyServe ) } + // Validate structured output here, while the client's own spelling is still + // known. The inner hop only sees the translated `output_config`, so letting + // it reject would name a field the OpenAI caller never sent. + if (rawBody.response_format !== undefined) { + const parsed = parseOutputFormat(anthropicBody.output_config, anthropicBody.tools, "openai") + if (!parsed.ok) { + return c.json( + { type: "error", error: { type: "invalid_request_error", message: parsed.message } }, + 400 + ) + } + } + // Route internally via app.fetch() — no network roundtrip. // Hono resolves the path in-process; the URL scheme/host are ignored. // Forward the caller's auth headers so requireAuth on /v1/messages accepts diff --git a/src/proxy/structuredOutput.ts b/src/proxy/structuredOutput.ts index 28f30997..aa2fc11b 100644 --- a/src/proxy/structuredOutput.ts +++ b/src/proxy/structuredOutput.ts @@ -17,8 +17,32 @@ function isRecord(value: unknown): value is Record { * mode buffers the SDK's wire events and replaces the response content with the * validated result, so a tool_use turn would be swallowed and the client-driven * tool loop would never see it. + * + * `dialect` selects which spelling errors name. The OpenAI endpoint translates + * `response_format` into this shape before validating, so it asks for its own + * field paths and errors point at what the client actually sent. */ -export function parseOutputFormat(outputConfig: unknown, tools?: unknown): OutputFormatParseResult { +export type OutputFormatDialect = "anthropic" | "openai" + +const ERROR_PATHS = { + anthropic: { + format: "output_config.format", + type: "output_config.format.type", + schema: "output_config.format.schema", + }, + openai: { + format: "response_format", + type: "response_format.type", + schema: "response_format.json_schema.schema", + }, +} as const + +export function parseOutputFormat( + outputConfig: unknown, + tools?: unknown, + dialect: OutputFormatDialect = "anthropic", +): OutputFormatParseResult { + const paths = ERROR_PATHS[dialect] if (outputConfig === undefined) return { ok: true, value: undefined } if (!isRecord(outputConfig)) { return { ok: false, message: "output_config: Expected an object" } @@ -27,24 +51,45 @@ export function parseOutputFormat(outputConfig: unknown, tools?: unknown): Outpu const format = outputConfig.format if (format === undefined) return { ok: true, value: undefined } if (!isRecord(format)) { - return { ok: false, message: "output_config.format: Expected an object" } + return { ok: false, message: `${paths.format}: Expected an object` } } if (format.type !== "json_schema") { - return { ok: false, message: "output_config.format.type: Only 'json_schema' is supported" } + return { ok: false, message: `${paths.type}: Only 'json_schema' is supported` } } if (!isRecord(format.schema)) { - return { ok: false, message: "output_config.format.schema: Expected a JSON Schema object" } + return { ok: false, message: `${paths.schema}: Expected a JSON Schema object` } } if (Array.isArray(tools) && tools.length > 0) { - return { ok: false, message: "output_config.format: Cannot be combined with tools" } + return { ok: false, message: `${paths.format}: Cannot be combined with tools` } } return { ok: true, - value: { type: "json_schema", schema: format.schema }, + value: { type: "json_schema", schema: stripRootSchemaKeyword(format.schema) }, } } +/** + * Drop a root-level `$schema` before handing the schema to the SDK. + * + * Anything but the draft-07 URI makes the model fail to submit its structured + * result: the request burns its turn budget and ends with no structured_output, + * surfacing as a 500. That includes the 2020-12 URI, which zod v4's + * `z.toJSONSchema()` emits by default - so every schema from the zod/Vercel AI + * SDK toolchain hits it. + * + * The keyword only declares which dialect the schema is written in and + * constrains nothing, so dropping it changes no validation semantics. Nested + * occurrences are left alone: they are already tolerated, and rewriting a + * caller's schema deeper than necessary risks touching a `properties` key + * legitimately named `$schema`. + */ +function stripRootSchemaKeyword(schema: Record): Record { + if (!("$schema" in schema)) return schema + const { $schema: _dialect, ...rest } = schema + return rest +} + export function structuredOutputText(value: unknown): string { return JSON.stringify(value) } From ded55dfb837f00d5cd2d4e38a2fd708faa1d121f Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Thu, 3 Sep 2026 17:33:01 -0600 Subject: [PATCH 2/4] fix(openai): treat a null response_format as omission, not a crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many OpenAI-compatible clients serialize an unset optional as JSON null rather than dropping the key. translateResponseFormat read `.type` off the value before any validation, so `"response_format": null` threw a TypeError out of the /v1/chat/completions handler, which has no try/catch around the translation — turning a plain chat request that worked before structured output existed into a 500. Treat null as omission, and forward a non-object response_format untouched so parseOutputFormat rejects it with a 400 naming the client's own field instead of it being silently ignored. The boundary check in server.ts skips null for the same reason, which also keeps the error dialect honest: a null alongside an Anthropic-style output_config.format would otherwise report failures against `response_format.*`, a field the client did not meaningfully send. --- src/__tests__/openai.test.ts | 31 +++++++++++++++++++++++ src/__tests__/proxy-openai-compat.test.ts | 18 +++++++++++++ src/proxy/openai.ts | 21 +++++++++++---- src/proxy/server.ts | 7 ++++- 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/src/__tests__/openai.test.ts b/src/__tests__/openai.test.ts index 198b88a0..4d7fa36e 100644 --- a/src/__tests__/openai.test.ts +++ b/src/__tests__/openai.test.ts @@ -794,6 +794,37 @@ describe("translateOpenAiToAnthropic", () => { }) expect(result!.output_config).toBeUndefined() }) + + // Clients that serialize an unset optional as JSON null rather than omitting + // the key must behave as if it were absent. This threw a TypeError out of the + // /v1/chat/completions handler, which has no try/catch — a 500 on a request + // that worked fine before structured output landed. + it("treats an explicit null response_format as omission", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: null, + } as unknown as Parameters[0]) + expect(result!.output_config).toBeUndefined() + }) + + it("keeps output_config.effort when response_format is null", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: null, + output_config: { effort: "high" }, + } as unknown as Parameters[0]) + expect(result!.output_config).toEqual({ effort: "high" }) + }) + + // Forwarded rather than dropped, so the boundary check rejects it with a 400 + // naming the client's own field instead of silently ignoring the request. + it("forwards a non-object response_format for rejection downstream", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: "json_schema", + } as unknown as Parameters[0]) + expect(result!.output_config?.format).toBe("json_schema") + }) }) // --------------------------------------------------------------------------- diff --git a/src/__tests__/proxy-openai-compat.test.ts b/src/__tests__/proxy-openai-compat.test.ts index 74ea097b..873c367b 100644 --- a/src/__tests__/proxy-openai-compat.test.ts +++ b/src/__tests__/proxy-openai-compat.test.ts @@ -220,6 +220,24 @@ describe("POST /v1/chat/completions — non-streaming", () => { expect(capturedOptions?.outputFormat).toEqual({ type: "json_schema", schema }) }) + it("serves a request whose response_format is an explicit null", async () => { + // Many OpenAI-compatible clients emit `"response_format": null` for an + // unset optional instead of omitting the key. Reading `.type` off it threw + // out of this handler, which has no try/catch — so a plain chat request + // that worked before structured output existed came back 500. + mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: false, + response_format: null, + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(200) + expect(capturedOptions?.outputFormat).toBeUndefined() + }) + it("rejects response_format json_object instead of silently ignoring it", async () => { // Anthropic has no schema-less JSON mode, so the request cannot be honored. // Failing loudly beats returning prose to a client expecting JSON. diff --git a/src/proxy/openai.ts b/src/proxy/openai.ts index 71bc38c9..72c790b7 100644 --- a/src/proxy/openai.ts +++ b/src/proxy/openai.ts @@ -492,14 +492,25 @@ function summarizeAnthropicContent(content: string | AnthropicContentBlock[]): s * accepting it silently would promise an enforcement the request never gets. * parseOutputFormat rejects it with an actionable message. */ -function translateResponseFormat(format: OpenAiResponseFormat | undefined): unknown { - if (format === undefined || format.type === "text") return undefined +function translateResponseFormat(format: unknown): unknown { + // An explicit JSON `null` must behave exactly like omission. Plenty of + // OpenAI-compatible clients serialize an unset optional as `null` rather than + // dropping the key, and this runs before any validation: reading `.type` off + // it threw a TypeError out of a handler with no try/catch, turning a request + // that worked before structured output existed into a 500. + if (format === undefined || format === null) return undefined + // Anything that is not an object is forwarded untouched so parseOutputFormat + // rejects it with a 400 naming the client's own field, rather than being + // silently ignored here. + if (typeof format !== "object") return format + const shape = format as OpenAiResponseFormat + if (shape.type === "text") return undefined // `name` is a client-side label; `strict` has no equivalent - the SDK always // validates, which is never weaker than strict asked for. - if (format.type === "json_schema") { - return { type: "json_schema", schema: format.json_schema?.schema } + if (shape.type === "json_schema") { + return { type: "json_schema", schema: shape.json_schema?.schema } } - return { type: format.type } + return { type: shape.type } } export function translateOpenAiToAnthropic( diff --git a/src/proxy/server.ts b/src/proxy/server.ts index bfbea149..ada9174a 100644 --- a/src/proxy/server.ts +++ b/src/proxy/server.ts @@ -7017,7 +7017,12 @@ export function createProxyServer(config: Partial = {}): ProxyServe // Validate structured output here, while the client's own spelling is still // known. The inner hop only sees the translated `output_config`, so letting // it reject would name a field the OpenAI caller never sent. - if (rawBody.response_format !== undefined) { + // `null` is omission, not a request for structured output — see + // translateResponseFormat. Skipping it also keeps the error dialect honest: + // a null alongside an Anthropic-style `output_config.format` would + // otherwise report failures against `response_format.*`, a field the client + // did not meaningfully send. + if (rawBody.response_format !== undefined && rawBody.response_format !== null) { const parsed = parseOutputFormat(anthropicBody.output_config, anthropicBody.tools, "openai") if (!parsed.ok) { return c.json( From af4defa626fa1cfb642ca4db460e74909971721f Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Thu, 3 Sep 2026 19:01:27 -0600 Subject: [PATCH 3/4] test(openai): cover the structured-output response path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing test asserts only what reaches the SDK, and its request actually ends in a 500: the mock yields no result carrying structured_output, so nothing downstream of the SDK boundary runs. The feature's whole point — schema-valid JSON in the response — had no coverage in either mode. Add a non-streaming case asserting the parsed JSON in choices[0].message.content, and a streaming case asserting the same value reassembled from the content deltas. Also restore translateOpenAiToAnthropic's JSDoc, which was left stranded above translateResponseFormat when that helper was inserted, so the "returns null if the request has no messages" contract that server.ts relies on documented the wrong function. --- src/__tests__/proxy-openai-compat.test.ts | 59 +++++++++++++++++++++++ src/proxy/openai.ts | 12 ++--- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/__tests__/proxy-openai-compat.test.ts b/src/__tests__/proxy-openai-compat.test.ts index 873c367b..27456a05 100644 --- a/src/__tests__/proxy-openai-compat.test.ts +++ b/src/__tests__/proxy-openai-compat.test.ts @@ -220,6 +220,35 @@ describe("POST /v1/chat/completions — non-streaming", () => { expect(capturedOptions?.outputFormat).toEqual({ type: "json_schema", schema }) }) + // The test above pins what reaches the SDK, but its request actually ends in + // a 500: the mock yields no `result` carrying structured_output, so nothing + // downstream of the SDK boundary is exercised. These two supply that result + // and assert the bytes the client receives — otherwise the whole point of the + // feature (schema-valid JSON in the response) has no coverage. + it("returns the validated JSON as the message content", async () => { + const schema = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + additionalProperties: false, + } + mockMessages = [ + assistantMessage([{ type: "text", text: "ignored prose" }]), + { type: "result", subtype: "success", is_error: false, structured_output: { answer: "42" } }, + ] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: false, + response_format: { type: "json_schema", json_schema: { name: "answer", schema } }, + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(200) + const body = await res.json() as { choices: Array<{ message: { content: string } }> } + expect(JSON.parse(body.choices[0]!.message.content)).toEqual({ answer: "42" }) + }) + it("serves a request whose response_format is an explicit null", async () => { // Many OpenAI-compatible clients emit `"response_format": null` for an // unset optional instead of omitting the key. Reading `.type` off it threw @@ -794,6 +823,36 @@ describe("POST /v1/chat/completions — streaming", () => { ?.index expect(startIndex).toBe(0) }) + + it("streams the validated JSON as a single content delta", async () => { + const schema = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + additionalProperties: false, + } + mockMessages = [ + messageStart("msg_1"), textBlockStart(0), textDelta(0, "ignored prose"), + blockStop(0), messageDelta("end_turn"), messageStop(), + { type: "result", subtype: "success", is_error: false, structured_output: { answer: "42" } }, + ] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: true, + response_format: { type: "json_schema", json_schema: { name: "answer", schema } }, + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(200) + const text = await readStream(res) + const content = text.split("\n") + .filter(l => l.startsWith("data: ") && l !== "data: [DONE]") + .map(l => JSON.parse(l.slice(6)) as { choices?: Array<{ delta?: { content?: string } }> }) + .map(c => c.choices?.[0]?.delta?.content ?? "") + .join("") + expect(JSON.parse(content)).toEqual({ answer: "42" }) + }) }) // --------------------------------------------------------------------------- diff --git a/src/proxy/openai.ts b/src/proxy/openai.ts index 72c790b7..e0935533 100644 --- a/src/proxy/openai.ts +++ b/src/proxy/openai.ts @@ -478,12 +478,6 @@ function summarizeAnthropicContent(content: string | AnthropicContentBlock[]): s // Request translation: OpenAI → Anthropic // --------------------------------------------------------------------------- -/** - * Translate an OpenAI /v1/chat/completions request body into an Anthropic - * /v1/messages request body. - * - * Returns null if the request has no messages (caller should return 400). - */ /** * Map OpenAI's `response_format` onto Anthropic's `output_config.format`. * @@ -513,6 +507,12 @@ function translateResponseFormat(format: unknown): unknown { return { type: shape.type } } +/** + * Translate an OpenAI /v1/chat/completions request body into an Anthropic + * /v1/messages request body. + * + * Returns null if the request has no messages (caller should return 400). + */ export function translateOpenAiToAnthropic( body: OpenAiChatRequest, options: OpenAiTranslationOptions = {}, From 1b3f94a3288259b677437e88ca7c2ef4175aafd7 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Thu, 3 Sep 2026 21:51:12 -0600 Subject: [PATCH 4/4] fix(openai): keep tool calling when a schema cannot also be honoured Structured-output mode buffers the SDK's wire events and replaces the response content, so it cannot coexist with a client-driven tool loop -- the tool_use turn gets swallowed. /v1/messages has always rejected the combination and continues to; nothing has depended on it working there. /v1/chat/completions is different. It dropped response_format entirely before structured output existed, so tool calling worked, and OpenAI permits both fields together -- LangChain agents and LiteLLM send them routinely. Routing those requests into the same rejection turned a working tool loop into a 400. Reject only when nothing the caller asked for can be honoured. With tools present the tools are honoured and the unsatisfiable schema is dropped and logged. json_object on its own stays a 400: there is no schema-less JSON mode to fall back to, and handing prose to a caller about to JSON.parse it is a worse failure than an actionable error. --- src/__tests__/proxy-openai-compat.test.ts | 32 ++++++++++++++++++++--- src/proxy/server.ts | 17 ++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/__tests__/proxy-openai-compat.test.ts b/src/__tests__/proxy-openai-compat.test.ts index 27456a05..d72d6f7c 100644 --- a/src/__tests__/proxy-openai-compat.test.ts +++ b/src/__tests__/proxy-openai-compat.test.ts @@ -282,9 +282,12 @@ describe("POST /v1/chat/completions — non-streaming", () => { expect(res.status).toBe(400) }) - it("rejects response_format combined with tools", async () => { - // Structured-output mode replaces the response content, which would swallow - // a tool_use turn and strand the client's tool loop. + it("keeps tool calling and drops the schema when both are sent", async () => { + // OpenAI permits tools + response_format; structured-output mode cannot + // honour both, because it replaces the content and swallows the tool_use + // turn. This endpoint dropped response_format entirely before structured + // output existed, so tool calling worked and clients depend on it — a 400 + // delivers neither capability, dropping the schema delivers the larger one. mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] const app = createTestApp() @@ -298,7 +301,28 @@ describe("POST /v1/chat/completions — non-streaming", () => { messages: [{ role: "user", content: "Hi" }], }) - expect(res.status).toBe(400) + expect(res.status).toBe(200) + // The tools still reach the SDK; only the unsatisfiable schema is dropped. + expect(capturedOptions?.outputFormat).toBeUndefined() + }) + + // Same rule applied to json_object: on its own it is a 400, because nothing + // can be honoured (see "rejects response_format json_object" above, and note + // Anthropic has no schema-less JSON mode). Sent alongside tools there IS + // something to honour, so it degrades rather than failing the whole request. + it("keeps tool calling when json_object is sent alongside tools", async () => { + mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: false, + response_format: { type: "json_object" }, + tools: [{ type: "function", function: { name: "fn", parameters: {} } }], + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(200) + expect(capturedOptions?.outputFormat).toBeUndefined() }) it("sends the client system prompt verbatim, without the claude_code preset", async () => { diff --git a/src/proxy/server.ts b/src/proxy/server.ts index ada9174a..05eadf46 100644 --- a/src/proxy/server.ts +++ b/src/proxy/server.ts @@ -7023,6 +7023,23 @@ export function createProxyServer(config: Partial = {}): ProxyServe // otherwise report failures against `response_format.*`, a field the client // did not meaningfully send. if (rawBody.response_format !== undefined && rawBody.response_format !== null) { + // NOTE: agent-specific. OpenAI permits `tools` and `response_format` + // together; structured-output mode cannot honour both, because it buffers + // the wire events and replaces the content, swallowing the tool_use turn + // (see parseOutputFormat). /v1/messages 400s that combination and keeps + // doing so — nothing has ever depended on it working there. + // + // This endpoint is different: `response_format` was dropped entirely + // before structured output existed, so tool calling worked and clients + // (LangChain agents, LiteLLM) rely on it. 400ing them delivers neither + // capability; dropping the schema delivers the larger one. Reject only + // when nothing the caller asked for can be honoured. + const hasTools = Array.isArray(anthropicBody.tools) && anthropicBody.tools.length > 0 + if (hasTools && anthropicBody.output_config?.format !== undefined) { + const { format: _unsupportedWithTools, ...rest } = anthropicBody.output_config + anthropicBody.output_config = Object.keys(rest).length > 0 ? rest : undefined + claudeLog("openai.structured_output_dropped", { reason: "tools_present" }) + } const parsed = parseOutputFormat(anthropicBody.output_config, anthropicBody.tools, "openai") if (!parsed.ok) { return c.json(