diff --git a/src/__tests__/openai.test.ts b/src/__tests__/openai.test.ts index f431451d..4d7fa36e 100644 --- a/src/__tests__/openai.test.ts +++ b/src/__tests__/openai.test.ts @@ -732,6 +732,99 @@ describe("translateOpenAiToAnthropic", () => { expect(result!.tools).toHaveLength(1) expect(result!.tools![0]!.description).toBe("") }) + + // --- structured output --- + + const personSchema = { + type: "object", + properties: { name: { type: "string" } }, + required: ["name"], + } + + it("maps response_format json_schema to output_config.format", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: { + type: "json_schema", + json_schema: { name: "person", schema: personSchema, strict: true }, + }, + }) + // `name` and `strict` have no Anthropic equivalent and are dropped: the + // SDK always validates, which is never weaker than strict asked for. + expect(result!.output_config).toEqual({ + format: { type: "json_schema", schema: personSchema }, + }) + }) + + it("keeps response_format alongside output_config.effort", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + output_config: { effort: "high" }, + response_format: { type: "json_schema", json_schema: { schema: personSchema } }, + }) + expect(result!.output_config).toEqual({ + effort: "high", + format: { type: "json_schema", schema: personSchema }, + }) + }) + + it("passes through an Anthropic-shaped output_config.format unchanged", () => { + const format = { type: "json_schema", schema: personSchema } + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + output_config: { format }, + }) + expect(result!.output_config).toEqual({ format }) + }) + + it("forwards json_object intact so the boundary can reject it", () => { + // Not widened into a permissive schema: Anthropic has no schema-less JSON + // mode, and accepting it here would promise an enforcement never applied. + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: { type: "json_object" }, + }) + expect(result!.output_config).toEqual({ format: { type: "json_object" } }) + }) + + it("treats response_format text as no constraint", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: { type: "text" }, + }) + expect(result!.output_config).toBeUndefined() + }) + + // Clients that serialize an unset optional as JSON null rather than omitting + // the key must behave as if it were absent. This threw a TypeError out of the + // /v1/chat/completions handler, which has no try/catch — a 500 on a request + // that worked fine before structured output landed. + it("treats an explicit null response_format as omission", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: null, + } as unknown as Parameters[0]) + expect(result!.output_config).toBeUndefined() + }) + + it("keeps output_config.effort when response_format is null", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: null, + output_config: { effort: "high" }, + } as unknown as Parameters[0]) + expect(result!.output_config).toEqual({ effort: "high" }) + }) + + // Forwarded rather than dropped, so the boundary check rejects it with a 400 + // naming the client's own field instead of silently ignoring the request. + it("forwards a non-object response_format for rejection downstream", () => { + const result = translateOpenAiToAnthropic({ + messages: [{ role: "user", content: "hi" }], + response_format: "json_schema", + } as unknown as Parameters[0]) + expect(result!.output_config?.format).toBe("json_schema") + }) }) // --------------------------------------------------------------------------- diff --git a/src/__tests__/proxy-openai-compat.test.ts b/src/__tests__/proxy-openai-compat.test.ts index 64c65958..d72d6f7c 100644 --- a/src/__tests__/proxy-openai-compat.test.ts +++ b/src/__tests__/proxy-openai-compat.test.ts @@ -198,6 +198,133 @@ describe("POST /v1/chat/completions — non-streaming", () => { expect(capturedOptions?.effort).toBe("high") }) + it("carries response_format json_schema through to the SDK outputFormat", async () => { + // Structured output is enforced by the SDK on the internal /v1/messages + // hop. Without this the field is dropped at the endpoint boundary and the + // client silently gets prose back instead of schema-valid JSON. + const schema = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + additionalProperties: false, + } + mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] + const app = createTestApp() + + await postChatCompletion(app, { + stream: false, + response_format: { type: "json_schema", json_schema: { name: "answer", schema } }, + messages: [{ role: "user", content: "Hi" }], + }) + + expect(capturedOptions?.outputFormat).toEqual({ type: "json_schema", schema }) + }) + + // The test above pins what reaches the SDK, but its request actually ends in + // a 500: the mock yields no `result` carrying structured_output, so nothing + // downstream of the SDK boundary is exercised. These two supply that result + // and assert the bytes the client receives — otherwise the whole point of the + // feature (schema-valid JSON in the response) has no coverage. + it("returns the validated JSON as the message content", async () => { + const schema = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + additionalProperties: false, + } + mockMessages = [ + assistantMessage([{ type: "text", text: "ignored prose" }]), + { type: "result", subtype: "success", is_error: false, structured_output: { answer: "42" } }, + ] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: false, + response_format: { type: "json_schema", json_schema: { name: "answer", schema } }, + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(200) + const body = await res.json() as { choices: Array<{ message: { content: string } }> } + expect(JSON.parse(body.choices[0]!.message.content)).toEqual({ answer: "42" }) + }) + + it("serves a request whose response_format is an explicit null", async () => { + // Many OpenAI-compatible clients emit `"response_format": null` for an + // unset optional instead of omitting the key. Reading `.type` off it threw + // out of this handler, which has no try/catch — so a plain chat request + // that worked before structured output existed came back 500. + mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: false, + response_format: null, + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(200) + expect(capturedOptions?.outputFormat).toBeUndefined() + }) + + it("rejects response_format json_object instead of silently ignoring it", async () => { + // Anthropic has no schema-less JSON mode, so the request cannot be honored. + // Failing loudly beats returning prose to a client expecting JSON. + mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: false, + response_format: { type: "json_object" }, + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(400) + }) + + it("keeps tool calling and drops the schema when both are sent", async () => { + // OpenAI permits tools + response_format; structured-output mode cannot + // honour both, because it replaces the content and swallows the tool_use + // turn. This endpoint dropped response_format entirely before structured + // output existed, so tool calling worked and clients depend on it — a 400 + // delivers neither capability, dropping the schema delivers the larger one. + mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: false, + response_format: { + type: "json_schema", + json_schema: { schema: { type: "object", properties: {} } }, + }, + tools: [{ type: "function", function: { name: "fn", parameters: {} } }], + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(200) + // The tools still reach the SDK; only the unsatisfiable schema is dropped. + expect(capturedOptions?.outputFormat).toBeUndefined() + }) + + // Same rule applied to json_object: on its own it is a 400, because nothing + // can be honoured (see "rejects response_format json_object" above, and note + // Anthropic has no schema-less JSON mode). Sent alongside tools there IS + // something to honour, so it degrades rather than failing the whole request. + it("keeps tool calling when json_object is sent alongside tools", async () => { + mockMessages = [assistantMessage([{ type: "text", text: "ok" }])] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: false, + response_format: { type: "json_object" }, + tools: [{ type: "function", function: { name: "fn", parameters: {} } }], + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(200) + expect(capturedOptions?.outputFormat).toBeUndefined() + }) + it("sends the client system prompt verbatim, without the claude_code preset", async () => { // The OpenAI endpoint serves generic chat clients (Open WebUI, curl). // Their system prompt must reach the SDK as a plain string — NOT wrapped @@ -720,6 +847,36 @@ describe("POST /v1/chat/completions — streaming", () => { ?.index expect(startIndex).toBe(0) }) + + it("streams the validated JSON as a single content delta", async () => { + const schema = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + additionalProperties: false, + } + mockMessages = [ + messageStart("msg_1"), textBlockStart(0), textDelta(0, "ignored prose"), + blockStop(0), messageDelta("end_turn"), messageStop(), + { type: "result", subtype: "success", is_error: false, structured_output: { answer: "42" } }, + ] + const app = createTestApp() + + const res = await postChatCompletion(app, { + stream: true, + response_format: { type: "json_schema", json_schema: { name: "answer", schema } }, + messages: [{ role: "user", content: "Hi" }], + }) + + expect(res.status).toBe(200) + const text = await readStream(res) + const content = text.split("\n") + .filter(l => l.startsWith("data: ") && l !== "data: [DONE]") + .map(l => JSON.parse(l.slice(6)) as { choices?: Array<{ delta?: { content?: string } }> }) + .map(c => c.choices?.[0]?.delta?.content ?? "") + .join("") + expect(JSON.parse(content)).toEqual({ answer: "42" }) + }) }) // --------------------------------------------------------------------------- diff --git a/src/__tests__/structured-output-parse.test.ts b/src/__tests__/structured-output-parse.test.ts new file mode 100644 index 00000000..014b357b --- /dev/null +++ b/src/__tests__/structured-output-parse.test.ts @@ -0,0 +1,62 @@ +/** + * Unit tests for src/proxy/structuredOutput.ts - pure parsing/normalization. + * No I/O, no mocks required. + */ + +import { describe, it, expect } from "bun:test" +import { parseOutputFormat } from "../proxy/structuredOutput" + +const schema = { + type: "object", + additionalProperties: false, + required: ["a"], + properties: { a: { type: "integer" } }, +} + +function parseSchema(input: Record) { + const result = parseOutputFormat({ format: { type: "json_schema", schema: input } }) + if (!result.ok) throw new Error(`expected ok, got: ${result.message}`) + return result.value as { type: "json_schema"; schema: Record } +} + +describe("parseOutputFormat - $schema handling", () => { + it("strips a root-level $schema before reaching the SDK", () => { + // Any dialect but draft-07 makes the model fail to submit its structured + // result, which surfaces as a 500. The keyword constrains nothing, so it + // is dropped rather than passed through. + const value = parseSchema({ + $schema: "https://json-schema.org/draft/2020-12/schema", + ...schema, + }) + expect(value.schema).toEqual(schema) + expect("$schema" in value.schema).toBe(false) + }) + + it("strips the draft-07 $schema too, for one consistent shape", () => { + const value = parseSchema({ $schema: "http://json-schema.org/draft-07/schema#", ...schema }) + expect(value.schema).toEqual(schema) + }) + + it("leaves a schema without $schema untouched", () => { + const value = parseSchema({ ...schema }) + expect(value.schema).toEqual(schema) + }) + + it("does not mutate the caller's schema object", () => { + const input = { $schema: "https://json-schema.org/draft/2020-12/schema", ...schema } + parseSchema(input) + expect(input.$schema).toBe("https://json-schema.org/draft/2020-12/schema") + }) + + it("keeps a nested $schema, which the SDK already tolerates", () => { + // Only the root keyword breaks submission. Rewriting deeper would risk + // clobbering a property legitimately named `$schema`. + const nested = { + type: "object", + properties: { + p: { $schema: "https://json-schema.org/draft/2020-12/schema", type: "object" }, + }, + } + expect(parseSchema(nested).schema).toEqual(nested) + }) +}) diff --git a/src/proxy/openai.ts b/src/proxy/openai.ts index be6bdff3..e0935533 100644 --- a/src/proxy/openai.ts +++ b/src/proxy/openai.ts @@ -67,6 +67,15 @@ export interface OpenAiChatToolCustom { export type OpenAiChatTool = OpenAiChatToolFunction | OpenAiChatToolCustom +export interface OpenAiResponseFormat { + type: "text" | "json_object" | "json_schema" + json_schema?: { + name?: string + schema?: unknown + strict?: boolean + } +} + export interface OpenAiChatRequest { model?: string messages?: OpenAiMessage[] @@ -79,7 +88,9 @@ export interface OpenAiChatRequest { /** Standard OpenAI reasoning level (low/medium/high/…). */ reasoning_effort?: string /** Anthropic-style nesting some clients use. */ - output_config?: { effort?: string } + output_config?: { effort?: string; format?: unknown } + /** Standard OpenAI structured output (json_schema / json_object / text). */ + response_format?: OpenAiResponseFormat stream_options?: { include_usage?: boolean } } @@ -128,7 +139,7 @@ export interface AnthropicRequestBody { /** Reasoning effort carried from the OpenAI request so the internal * /v1/messages hop forwards it to the SDK (value gated by normalizeEffort). */ reasoning_effort?: string - output_config?: { effort?: string } + output_config?: { effort?: string; format?: unknown } } export interface AnthropicUsage { @@ -467,6 +478,35 @@ function summarizeAnthropicContent(content: string | AnthropicContentBlock[]): s // Request translation: OpenAI → Anthropic // --------------------------------------------------------------------------- +/** + * Map OpenAI's `response_format` onto Anthropic's `output_config.format`. + * + * `json_object` is forwarded intact rather than widened into a permissive + * `{"type":"object"}` schema: Anthropic has no schema-less JSON mode, and + * accepting it silently would promise an enforcement the request never gets. + * parseOutputFormat rejects it with an actionable message. + */ +function translateResponseFormat(format: unknown): unknown { + // An explicit JSON `null` must behave exactly like omission. Plenty of + // OpenAI-compatible clients serialize an unset optional as `null` rather than + // dropping the key, and this runs before any validation: reading `.type` off + // it threw a TypeError out of a handler with no try/catch, turning a request + // that worked before structured output existed into a 500. + if (format === undefined || format === null) return undefined + // Anything that is not an object is forwarded untouched so parseOutputFormat + // rejects it with a 400 naming the client's own field, rather than being + // silently ignored here. + if (typeof format !== "object") return format + const shape = format as OpenAiResponseFormat + if (shape.type === "text") return undefined + // `name` is a client-side label; `strict` has no equivalent - the SDK always + // validates, which is never weaker than strict asked for. + if (shape.type === "json_schema") { + return { type: "json_schema", schema: shape.json_schema?.schema } + } + return { type: shape.type } +} + /** * Translate an OpenAI /v1/chat/completions request body into an Anthropic * /v1/messages request body. @@ -622,7 +662,19 @@ export function translateOpenAiToAnthropic( // and OpenAI clients always run at the model default. Validation happens // downstream via normalizeEffort. if (body.reasoning_effort !== undefined) result.reasoning_effort = body.reasoning_effort - if (body.output_config?.effort !== undefined) result.output_config = { effort: body.output_config.effort } + + // Structured output. `response_format` is the standard OpenAI spelling; + // `output_config.format` is accepted too because some clients send the + // Anthropic shape at this endpoint. Both are forwarded unvalidated so the + // single check in parseOutputFormat rejects them at the HTTP boundary. + const outputFormat = translateResponseFormat(body.response_format) ?? body.output_config?.format + const effort = body.output_config?.effort + if (effort !== undefined || outputFormat !== undefined) { + const outputConfig: NonNullable = {} + if (effort !== undefined) outputConfig.effort = effort + if (outputFormat !== undefined) outputConfig.format = outputFormat + result.output_config = outputConfig + } return result } diff --git a/src/proxy/server.ts b/src/proxy/server.ts index 1d8177f2..05eadf46 100644 --- a/src/proxy/server.ts +++ b/src/proxy/server.ts @@ -7014,6 +7014,41 @@ export function createProxyServer(config: Partial = {}): ProxyServe ) } + // Validate structured output here, while the client's own spelling is still + // known. The inner hop only sees the translated `output_config`, so letting + // it reject would name a field the OpenAI caller never sent. + // `null` is omission, not a request for structured output — see + // translateResponseFormat. Skipping it also keeps the error dialect honest: + // a null alongside an Anthropic-style `output_config.format` would + // otherwise report failures against `response_format.*`, a field the client + // did not meaningfully send. + if (rawBody.response_format !== undefined && rawBody.response_format !== null) { + // NOTE: agent-specific. OpenAI permits `tools` and `response_format` + // together; structured-output mode cannot honour both, because it buffers + // the wire events and replaces the content, swallowing the tool_use turn + // (see parseOutputFormat). /v1/messages 400s that combination and keeps + // doing so — nothing has ever depended on it working there. + // + // This endpoint is different: `response_format` was dropped entirely + // before structured output existed, so tool calling worked and clients + // (LangChain agents, LiteLLM) rely on it. 400ing them delivers neither + // capability; dropping the schema delivers the larger one. Reject only + // when nothing the caller asked for can be honoured. + const hasTools = Array.isArray(anthropicBody.tools) && anthropicBody.tools.length > 0 + if (hasTools && anthropicBody.output_config?.format !== undefined) { + const { format: _unsupportedWithTools, ...rest } = anthropicBody.output_config + anthropicBody.output_config = Object.keys(rest).length > 0 ? rest : undefined + claudeLog("openai.structured_output_dropped", { reason: "tools_present" }) + } + const parsed = parseOutputFormat(anthropicBody.output_config, anthropicBody.tools, "openai") + if (!parsed.ok) { + return c.json( + { type: "error", error: { type: "invalid_request_error", message: parsed.message } }, + 400 + ) + } + } + // Route internally via app.fetch() — no network roundtrip. // Hono resolves the path in-process; the URL scheme/host are ignored. // Forward the caller's auth headers so requireAuth on /v1/messages accepts diff --git a/src/proxy/structuredOutput.ts b/src/proxy/structuredOutput.ts index 28f30997..aa2fc11b 100644 --- a/src/proxy/structuredOutput.ts +++ b/src/proxy/structuredOutput.ts @@ -17,8 +17,32 @@ function isRecord(value: unknown): value is Record { * mode buffers the SDK's wire events and replaces the response content with the * validated result, so a tool_use turn would be swallowed and the client-driven * tool loop would never see it. + * + * `dialect` selects which spelling errors name. The OpenAI endpoint translates + * `response_format` into this shape before validating, so it asks for its own + * field paths and errors point at what the client actually sent. */ -export function parseOutputFormat(outputConfig: unknown, tools?: unknown): OutputFormatParseResult { +export type OutputFormatDialect = "anthropic" | "openai" + +const ERROR_PATHS = { + anthropic: { + format: "output_config.format", + type: "output_config.format.type", + schema: "output_config.format.schema", + }, + openai: { + format: "response_format", + type: "response_format.type", + schema: "response_format.json_schema.schema", + }, +} as const + +export function parseOutputFormat( + outputConfig: unknown, + tools?: unknown, + dialect: OutputFormatDialect = "anthropic", +): OutputFormatParseResult { + const paths = ERROR_PATHS[dialect] if (outputConfig === undefined) return { ok: true, value: undefined } if (!isRecord(outputConfig)) { return { ok: false, message: "output_config: Expected an object" } @@ -27,24 +51,45 @@ export function parseOutputFormat(outputConfig: unknown, tools?: unknown): Outpu const format = outputConfig.format if (format === undefined) return { ok: true, value: undefined } if (!isRecord(format)) { - return { ok: false, message: "output_config.format: Expected an object" } + return { ok: false, message: `${paths.format}: Expected an object` } } if (format.type !== "json_schema") { - return { ok: false, message: "output_config.format.type: Only 'json_schema' is supported" } + return { ok: false, message: `${paths.type}: Only 'json_schema' is supported` } } if (!isRecord(format.schema)) { - return { ok: false, message: "output_config.format.schema: Expected a JSON Schema object" } + return { ok: false, message: `${paths.schema}: Expected a JSON Schema object` } } if (Array.isArray(tools) && tools.length > 0) { - return { ok: false, message: "output_config.format: Cannot be combined with tools" } + return { ok: false, message: `${paths.format}: Cannot be combined with tools` } } return { ok: true, - value: { type: "json_schema", schema: format.schema }, + value: { type: "json_schema", schema: stripRootSchemaKeyword(format.schema) }, } } +/** + * Drop a root-level `$schema` before handing the schema to the SDK. + * + * Anything but the draft-07 URI makes the model fail to submit its structured + * result: the request burns its turn budget and ends with no structured_output, + * surfacing as a 500. That includes the 2020-12 URI, which zod v4's + * `z.toJSONSchema()` emits by default - so every schema from the zod/Vercel AI + * SDK toolchain hits it. + * + * The keyword only declares which dialect the schema is written in and + * constrains nothing, so dropping it changes no validation semantics. Nested + * occurrences are left alone: they are already tolerated, and rewriting a + * caller's schema deeper than necessary risks touching a `properties` key + * legitimately named `$schema`. + */ +function stripRootSchemaKeyword(schema: Record): Record { + if (!("$schema" in schema)) return schema + const { $schema: _dialect, ...rest } = schema + return rest +} + export function structuredOutputText(value: unknown): string { return JSON.stringify(value) }