From 96e2376999befb14ef81584194ff15c33b7173a8 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 02:06:10 -0700 Subject: [PATCH] fix(openai): require nested strict schema fields Align the recursive strict JSON Schema contract with OpenAI by marking every nested object property required while representing Prompty optional fields as nullable. Add a shared wire vector, cross-runtime harness coverage, and a live nested structured-output check. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/rust/prompty-openai/src/wire.rs | 2 +- .../rust/prompty-openai/tests/integration.rs | 28 ++++++--- .../rust/prompty-openai/tests/wire_vectors.rs | 5 +- .../packages/core/tests/spec-vectors.test.ts | 14 +---- spec/spec.md | 17 ++++-- spec/vectors/wire/wire_vectors.json | 59 +++++++++++++++++++ .../docs/specification/wire-format.mdx | 14 ++++- 7 files changed, 106 insertions(+), 33 deletions(-) diff --git a/runtime/rust/prompty-openai/src/wire.rs b/runtime/rust/prompty-openai/src/wire.rs index e23512b1b..cf8072330 100644 --- a/runtime/rust/prompty-openai/src/wire.rs +++ b/runtime/rust/prompty-openai/src/wire.rs @@ -385,7 +385,7 @@ fn property_to_json_schema(prop: &Property, strict: bool) -> Result panic!("Expected object or JSON string, got: {other:?}"), }; - assert!(obj.contains_key("city"), "missing 'city' field: {obj:?}"); + let style = obj + .get("style") + .and_then(Value::as_object) + .expect("missing object 'style' field"); + assert!(style.get("color").is_some_and(Value::is_string)); assert!( - obj.contains_key("country"), - "missing 'country' field: {obj:?}" + style.contains_key("border"), + "missing 'border' field: {style:?}" ); assert!( - obj.contains_key("population"), - "missing 'population' field: {obj:?}" + style["border"].is_null() || style["border"].is_string(), + "'border' must be nullable: {style:?}" ); eprintln!("Structured output: {obj:?}"); } diff --git a/runtime/rust/prompty-openai/tests/wire_vectors.rs b/runtime/rust/prompty-openai/tests/wire_vectors.rs index fecbb32f2..ff1955b68 100644 --- a/runtime/rust/prompty-openai/tests/wire_vectors.rs +++ b/runtime/rust/prompty-openai/tests/wire_vectors.rs @@ -197,6 +197,7 @@ wire_test!(image_wire); wire_test!(kind_to_json_type_mapping); wire_test!(chat_image_part); wire_test!(structured_output); +wire_test!(structured_output_nested_optional); wire_test!(options_max_completion_tokens); wire_test!(options_stop_sequences); wire_test!(options_additional_properties); @@ -300,7 +301,7 @@ fn set_row_visual_schema_supports_nullable_unions_and_nested_optionality() { assert_eq!(schema["properties"]["row"]["type"], "object"); assert_eq!( schema["properties"]["row"]["required"], - json!(["color", "fill"]) + json!(["color", "border", "fill"]) ); assert_eq!( schema["properties"]["row"]["properties"]["color"]["type"], @@ -328,7 +329,7 @@ fn set_row_visual_schema_supports_nullable_unions_and_nested_optionality() { ); assert_eq!( schema["properties"]["row"]["properties"]["fill"]["anyOf"][1]["required"], - json!(["theme"]) + json!(["theme", "tint"]) ); assert_no_empty_type(&schema); } diff --git a/runtime/typescript/packages/core/tests/spec-vectors.test.ts b/runtime/typescript/packages/core/tests/spec-vectors.test.ts index fdc30b66c..3d52db073 100644 --- a/runtime/typescript/packages/core/tests/spec-vectors.test.ts +++ b/runtime/typescript/packages/core/tests/spec-vectors.test.ts @@ -751,12 +751,7 @@ function buildAgentFromWireInput(input: any, name?: string): Prompty { // Build tools if (input.tools && input.tools.length > 0) { agent.tools = input.tools.map((t: any) => { - const params = (t.parameters ?? []).map((p: any) => new Property({ - name: p.name, - kind: p.kind, - required: p.required, - description: p.description, - })); + const params = (t.parameters ?? []).map((p: any) => Property.load(p)); const bindings = Object.entries(t.bindings ?? {}).map(([bname, bval]: [string, any]) => new Binding({ name: bname, input: typeof bval === "object" ? bval.input : String(bval) }), ); @@ -773,12 +768,7 @@ function buildAgentFromWireInput(input: any, name?: string): Prompty { // Build outputs if (input.outputs && input.outputs.length > 0) { - agent.outputs = input.outputs.map((o: any) => new Property({ - name: o.name, - kind: o.kind, - required: o.required, - description: o.description, - })); + agent.outputs = input.outputs.map((o: any) => Property.load(o)); } return agent; diff --git a/spec/spec.md b/spec/spec.md index 67c554f69..4b6ef9635 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -1496,17 +1496,18 @@ A `list[Property]` (used for `inputs`, `outputs`, and `FunctionTool.parameters`) MUST be converted to a JSON Schema object for wire transmission: ``` -function schema_to_wire(properties: list[Property]) → dict: +function schema_to_wire(properties: list[Property], strict: bool = false) → dict: schema = { type: "object", properties: {}, required: [] } for prop in properties: - prop_schema = property_to_json_schema(prop) + optional = strict and not prop.required + prop_schema = property_to_json_schema(prop, strict, optional) if prop.description: prop_schema.description = prop.description if prop.enumValues: prop_schema.enum = prop.enumValues schema.properties[prop.name] = prop_schema - if prop.required: + if strict or prop.required: schema.required.append(prop.name) if schema.required is empty: @@ -1516,9 +1517,13 @@ function schema_to_wire(properties: list[Property]) → dict: ``` `property_to_json_schema` MUST recursively convert array items and object -properties. Object `required` arrays MUST contain only children whose -`Property.required` is true. For a concrete property with `nullable: true`, -the JSON Schema `type` MUST include both the concrete type and `"null"`. +properties. Outside provider strict mode, object `required` arrays MUST contain +only children whose `Property.required` is true. In OpenAI strict mode, this +rule MUST apply recursively: every property of every object MUST appear in that +object's `required` array, and a property whose `Property.required` is not true +MUST be represented as nullable. For a concrete property, nullability is +represented by a JSON Schema `type` containing both the concrete type and +`"null"`. `kind: "union"` represents a portable union property. Exactly one of its `oneOf` and `anyOf` arrays MUST be nonempty; implementations MUST reject diff --git a/spec/vectors/wire/wire_vectors.json b/spec/vectors/wire/wire_vectors.json index 8e28dd475..c62fe481f 100644 --- a/spec/vectors/wire/wire_vectors.json +++ b/spec/vectors/wire/wire_vectors.json @@ -435,6 +435,65 @@ } } }, + { + "name": "structured_output_nested_optional", + "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", + "input": { + "provider": "openai", + "apiType": "chat", + "model_id": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": [{ "kind": "text", "value": "Choose a visual style." }] + } + ], + "tools": [], + "options": {}, + "outputs": [ + { + "name": "style", + "kind": "object", + "required": true, + "properties": [ + { "name": "color", "kind": "string", "required": true }, + { "name": "border", "kind": "string", "required": false } + ] + } + ] + }, + "expected": { + "request_body": { + "model": "gpt-4o-mini", + "messages": [ + { "role": "user", "content": "Choose a visual style." } + ], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "structured_output", + "strict": true, + "schema": { + "type": "object", + "properties": { + "style": { + "type": "object", + "properties": { + "color": { "type": "string" }, + "border": { "type": ["string", "null"] } + }, + "required": ["color", "border"], + "additionalProperties": false + } + }, + "required": ["style"], + "additionalProperties": false + } + } + } + } + } + }, { "name": "kind_to_json_type_mapping", "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.", diff --git a/web/src/content/docs/specification/wire-format.mdx b/web/src/content/docs/specification/wire-format.mdx index 65418b715..f36bf6fd1 100644 --- a/web/src/content/docs/specification/wire-format.mdx +++ b/web/src/content/docs/specification/wire-format.mdx @@ -179,17 +179,18 @@ A `list[Property]` (used for `inputs`, `outputs`, and `FunctionTool.parameters`) MUST be converted to a JSON Schema object for wire transmission: ``` -function schema_to_wire(properties: list[Property]) → dict: +function schema_to_wire(properties: list[Property], strict: bool = false) → dict: schema = { type: "object", properties: {}, required: [] } for prop in properties: - prop_schema = { type: map_kind_to_json_type(prop.kind) } + optional = strict and not prop.required + prop_schema = property_to_json_schema(prop, strict, optional) if prop.description: prop_schema.description = prop.description if prop.enumValues: prop_schema.enum = prop.enumValues schema.properties[prop.name] = prop_schema - if prop.required: + if strict or prop.required: schema.required.append(prop.name) if schema.required is empty: @@ -198,6 +199,13 @@ function schema_to_wire(properties: list[Property]) → dict: return schema ``` +`property_to_json_schema` MUST recurse through array items and object +properties. Outside provider strict mode, each object lists only explicitly +required children in `required`. In OpenAI strict mode, every property of every +object MUST appear in that object's `required` array. A property that is +optional in the Prompty model MUST instead be represented as nullable in the +strict wire schema. + **Kind → JSON Schema type mapping.** Implementations MUST use this table: | Property `kind` | JSON Schema `type` |