From c96d27218875c475b3f20dcd90de3e9036214353 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Thu, 15 Oct 2026 21:15:44 -0700 Subject: [PATCH 1/3] fix(python): avoid shell release commands Run release git commands with argv lists so tag/version values are not interpreted by a shell. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 497512b4-8e78-461b-9e96-8e9af3fdb03c --- runtime/python/prompty/scripts/release.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/runtime/python/prompty/scripts/release.py b/runtime/python/prompty/scripts/release.py index 02137cef7..6634b9658 100644 --- a/runtime/python/prompty/scripts/release.py +++ b/runtime/python/prompty/scripts/release.py @@ -19,6 +19,7 @@ import argparse import re +import shlex import subprocess import sys from pathlib import Path @@ -76,9 +77,9 @@ def bump_version(current: str, bump: str) -> str: sys.exit(1) -def run(cmd: str) -> None: - print(f" $ {cmd}") - subprocess.run(cmd, cwd=REPO_ROOT, shell=True, check=True) +def run(cmd: list[str]) -> None: + print(f" $ {shlex.join(cmd)}") + subprocess.run(cmd, cwd=REPO_ROOT, check=True) def main() -> None: @@ -116,12 +117,12 @@ def main() -> None: # Git commit, tag, push print("\nšŸ”– Creating commit and tag...\n") - run("git add runtime/python/prompty/prompty/_version.py") - run(f'git commit -m "chore(python): release v{new_version}"') - run(f'git tag -a {tag} -m "{tag}"') + run(["git", "add", "runtime/python/prompty/prompty/_version.py"]) + run(["git", "commit", "-m", f"chore(python): release v{new_version}"]) + run(["git", "tag", "-a", tag, "-m", tag]) print("\nšŸš€ Pushing to origin...\n") - run("git push origin main --follow-tags") + run(["git", "push", "origin", "main", "--follow-tags"]) print(f"\nāœ… Done! Tag {tag} pushed.") print(" CI will build, test, and publish to PyPI.\n") From 8ddc2b559cfc254fa5d3b0b2b6d7d67875b29298 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Thu, 15 Oct 2026 20:52:12 -0700 Subject: [PATCH 2/3] fix(openai): extract nested Responses message text Fall back to completed Responses message content when the top-level output_text convenience field is missing or empty. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 497512b4-8e78-461b-9e96-8e9af3fdb03c --- .../Model/ProcessorConformanceTests.cs | 105 ++++++++++ runtime/go/prompty/model/process_response.go | 28 ++- .../go/prompty/model/process_response_test.go | 33 ++++ .../model/processor_conformance_test.go | 145 ++++++++++++++ .../prompty/openai/OpenAIProcessor.java | 25 ++- .../prompty/openai/ProcessVectorsTest.java | 44 +++++ .../model/ProcessorConformanceTests.java | 46 ++++- .../prompty/providers/openai/processor.py | 2 +- .../tests/model/test_processor_conformance.py | 107 ++++++++++ runtime/rust/prompty-openai/src/processor.rs | 75 +++++-- .../prompty-openai/tests/process_vectors.rs | 2 + .../tests/model/processor_conformance_test.rs | 127 ++++++++++++ .../PromptyModel/PipelineRuntime.swift | 19 +- .../PipelineRuntimeTests.swift | 31 +++ .../ProcessorConformanceTests.swift | 44 ++++- .../tests/model/processor.conformance.test.ts | 24 +++ .../packages/openai/src/processor.ts | 2 +- .../packages/openai/tests/kinds.test.ts | 26 ++- schema/model/conformance/vectors/process.tsp | 81 ++++++++ .../tsp-output/.typra-generated/vectors.json | 185 ++++++++++++++++++ 20 files changed, 1114 insertions(+), 37 deletions(-) create mode 100644 runtime/go/prompty/model/process_response_test.go create mode 100644 runtime/swift/prompty-model/Tests/PromptyModelTests/PipelineRuntimeTests.swift diff --git a/runtime/csharp/Prompty.Core.Tests/Model/ProcessorConformanceTests.cs b/runtime/csharp/Prompty.Core.Tests/Model/ProcessorConformanceTests.cs index 4d3e4143e..849c06f2e 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/ProcessorConformanceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/ProcessorConformanceTests.cs @@ -906,6 +906,111 @@ public async Task ResponsesEmptyOutput() Assert.NotNull(actual); } + [Fact] + public async Task ResponsesEmptyOutputTextMessageContentFallback() + { + string inputJson = """ +{ + "agent": { + "model": { + "provider": "openai" + } + }, + "apiType": "responses", + "response": { + "id": "resp_empty_output_text_fallback", + "object": "response", + "created_at": 1700000024, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_empty_output_text_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Nested text wins.", + "annotations": [] + } + ] + } + ], + "output_text": "", + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } + }, + "has_outputs": false +} +"""; + using var document = JsonDocument.Parse(inputJson); + var root = document.RootElement; + var agent = Agent.FromJson(root.GetProperty("agent").GetRawText()); + var response = JsonSerializer.Deserialize(root.GetProperty("response").GetRawText()); + var provider = agent.Model.Provider; + var impl = ProcessorResolver.Resolve(provider, Provider()); + Assert.NotNull(impl); + var actual = await impl!.ProcessAsync(agent, response); + Assert.NotNull(actual); + } + + [Fact] + public async Task ResponsesMessageContentFallback() + { + string inputJson = """ +{ + "agent": { + "model": { + "provider": "openai" + } + }, + "apiType": "responses", + "response": { + "id": "resp_content_fallback", + "object": "response", + "created_at": 1700000023, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_content_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Visible final answer.", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } + }, + "has_outputs": false +} +"""; + using var document = JsonDocument.Parse(inputJson); + var root = document.RootElement; + var agent = Agent.FromJson(root.GetProperty("agent").GetRawText()); + var response = JsonSerializer.Deserialize(root.GetProperty("response").GetRawText()); + var provider = agent.Model.Provider; + var impl = ProcessorResolver.Resolve(provider, Provider()); + Assert.NotNull(impl); + var actual = await impl!.ProcessAsync(agent, response); + Assert.NotNull(actual); + } + [Fact] public async Task ResponsesStructured() { diff --git a/runtime/go/prompty/model/process_response.go b/runtime/go/prompty/model/process_response.go index 25653cab3..a57bf487b 100644 --- a/runtime/go/prompty/model/process_response.go +++ b/runtime/go/prompty/model/process_response.go @@ -100,10 +100,36 @@ func processOpenAIResponses(resp map[string]interface{}, hasOutputs bool) any { if len(toolCalls) > 0 { return toolCalls } - outputText, _ := resp["output_text"].(string) + outputText := openAIResponsesOutputText(resp) return maybeParseStructured(outputText, hasOutputs) } +func openAIResponsesOutputText(resp map[string]interface{}) string { + if outputText, _ := resp["output_text"].(string); outputText != "" { + return outputText + } + + output, _ := resp["output"].([]interface{}) + var b strings.Builder + for _, item := range output { + it, _ := item.(map[string]interface{}) + if it["type"] != "message" { + continue + } + content, _ := it["content"].([]interface{}) + for _, part := range content { + p, _ := part.(map[string]interface{}) + partType, _ := p["type"].(string) + if partType != "output_text" && partType != "text" { + continue + } + text, _ := p["text"].(string) + b.WriteString(text) + } + } + return b.String() +} + func processOpenAIEmbedding(resp map[string]interface{}) any { data, _ := resp["data"].([]interface{}) if len(data) == 1 { diff --git a/runtime/go/prompty/model/process_response_test.go b/runtime/go/prompty/model/process_response_test.go new file mode 100644 index 000000000..07ada6786 --- /dev/null +++ b/runtime/go/prompty/model/process_response_test.go @@ -0,0 +1,33 @@ +package prompty_test + +import ( + "testing" + + prompty "prompty/model" +) + +func TestOpenAIResponsesEmptyOutputTextFallsBackToMessageContent(t *testing.T) { + response := map[string]interface{}{ + "object": "response", + "output_text": "", + "output": []interface{}{ + map[string]interface{}{ + "type": "message", + "content": []interface{}{ + map[string]interface{}{ + "type": "output_text", + "text": "Nested text wins.", + }, + }, + }, + }, + } + + result, err := prompty.ProcessResponse("openai", "responses", response, false) + if err != nil { + t.Fatalf("ProcessResponse failed: %v", err) + } + if result != "Nested text wins." { + t.Fatalf("expected nested text fallback, got %#v", result) + } +} diff --git a/runtime/go/prompty/model/processor_conformance_test.go b/runtime/go/prompty/model/processor_conformance_test.go index da8bd56fd..e6de5616f 100644 --- a/runtime/go/prompty/model/processor_conformance_test.go +++ b/runtime/go/prompty/model/processor_conformance_test.go @@ -1283,6 +1283,151 @@ func TestProcessorProcessResponsesEmptyOutput(t *testing.T) { _ = actual } +func TestProcessorProcessResponsesEmptyOutputTextMessageContentFallback(t *testing.T) { + inputJSON := `{ + "agent": { + "model": { + "provider": "openai" + } + }, + "apiType": "responses", + "response": { + "id": "resp_empty_output_text_fallback", + "object": "response", + "created_at": 1700000024, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_empty_output_text_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Nested text wins.", + "annotations": [] + } + ] + } + ], + "output_text": "", + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } + }, + "has_outputs": false +}` + var payload map[string]any + if err := json.Unmarshal([]byte(inputJSON), &payload); err != nil { + t.Fatalf("failed to decode vector input: %v", err) + } + agent, err := fixtures.LoadAgent(payload["agent"], fixtures.NewLoadContext()) + if err != nil { + t.Fatalf("agent parse: %v", err) + } + var response interface{} + { + responseBytes, marshalErr := json.Marshal(payload["response"]) + if marshalErr != nil { + t.Fatalf("response marshal: %v", marshalErr) + } + if unmarshalErr := json.Unmarshal(responseBytes, &response); unmarshalErr != nil { + t.Fatalf("response parse: %v", unmarshalErr) + } + } + provider := agent.Model.(interface { + Save(*fixtures.SaveContext) map[string]interface{} + }).Save(fixtures.NewSaveContext())["provider"].(string) + impl, err := fixtures.ResolveProcessor(provider, vectoradapters.ProcessorProvider()) + if err != nil { + t.Fatalf("resolve %q: %v", provider, err) + } + if impl == nil { + t.Fatalf("responses_empty_output_text_message_content_fallback: no Processor attached for %q", provider) + } + actual, err := impl.Process(agent, response) + if err != nil { + t.Fatalf("responses_empty_output_text_message_content_fallback: %v", err) + } + _ = actual +} + +func TestProcessorProcessResponsesMessageContentFallback(t *testing.T) { + inputJSON := `{ + "agent": { + "model": { + "provider": "openai" + } + }, + "apiType": "responses", + "response": { + "id": "resp_content_fallback", + "object": "response", + "created_at": 1700000023, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_content_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Visible final answer.", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } + }, + "has_outputs": false +}` + var payload map[string]any + if err := json.Unmarshal([]byte(inputJSON), &payload); err != nil { + t.Fatalf("failed to decode vector input: %v", err) + } + agent, err := fixtures.LoadAgent(payload["agent"], fixtures.NewLoadContext()) + if err != nil { + t.Fatalf("agent parse: %v", err) + } + var response interface{} + { + responseBytes, marshalErr := json.Marshal(payload["response"]) + if marshalErr != nil { + t.Fatalf("response marshal: %v", marshalErr) + } + if unmarshalErr := json.Unmarshal(responseBytes, &response); unmarshalErr != nil { + t.Fatalf("response parse: %v", unmarshalErr) + } + } + provider := agent.Model.(interface { + Save(*fixtures.SaveContext) map[string]interface{} + }).Save(fixtures.NewSaveContext())["provider"].(string) + impl, err := fixtures.ResolveProcessor(provider, vectoradapters.ProcessorProvider()) + if err != nil { + t.Fatalf("resolve %q: %v", provider, err) + } + if impl == nil { + t.Fatalf("responses_message_content_fallback: no Processor attached for %q", provider) + } + actual, err := impl.Process(agent, response) + if err != nil { + t.Fatalf("responses_message_content_fallback: %v", err) + } + _ = actual +} + func TestProcessorProcessResponsesStructured(t *testing.T) { inputJSON := `{ "agent": { diff --git a/runtime/java/prompty-openai/src/main/java/com/microsoft/prompty/openai/OpenAIProcessor.java b/runtime/java/prompty-openai/src/main/java/com/microsoft/prompty/openai/OpenAIProcessor.java index 1ac4149c8..d73aa5977 100644 --- a/runtime/java/prompty-openai/src/main/java/com/microsoft/prompty/openai/OpenAIProcessor.java +++ b/runtime/java/prompty-openai/src/main/java/com/microsoft/prompty/openai/OpenAIProcessor.java @@ -164,7 +164,7 @@ private List portableAssistantMessages(Object response) { if (!messages.isEmpty()) { return messages; } - messages.add(Messages.assistant(stringOrEmpty(Streams.pointer(response, "output_text")))); + messages.add(Messages.assistant(responsesOutputText(response))); return messages; } @@ -280,7 +280,28 @@ private static Object processResponsesApi(Agent agent, Object response) { if (!toolCalls.isEmpty()) { return toolCalls; } - return structuredOrText(agent, stringOrEmpty(Streams.pointer(response, "output_text"))); + return structuredOrText(agent, responsesOutputText(response)); + } + + private static String responsesOutputText(Object response) { + String outputText = stringOrEmpty(Streams.pointer(response, "output_text")); + if (!outputText.isEmpty()) { + return outputText; + } + + StringBuilder text = new StringBuilder(); + for (Object item : asList(Streams.pointer(response, "output"))) { + if (!"message".equals(Streams.pointer(item, "type"))) { + continue; + } + for (Object part : asList(Streams.pointer(item, "content"))) { + Object partType = Streams.pointer(part, "type"); + if ("output_text".equals(partType) || "text".equals(partType)) { + text.append(stringOrEmpty(Streams.pointer(part, "text"))); + } + } + } + return text.toString(); } /** diff --git a/runtime/java/prompty-openai/src/test/java/com/microsoft/prompty/openai/ProcessVectorsTest.java b/runtime/java/prompty-openai/src/test/java/com/microsoft/prompty/openai/ProcessVectorsTest.java index d05d75a64..283ca3b02 100644 --- a/runtime/java/prompty-openai/src/test/java/com/microsoft/prompty/openai/ProcessVectorsTest.java +++ b/runtime/java/prompty-openai/src/test/java/com/microsoft/prompty/openai/ProcessVectorsTest.java @@ -1,12 +1,20 @@ package com.microsoft.prompty.openai; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.microsoft.prompty.Messages; import com.microsoft.prompty.SpecVectors; import com.microsoft.prompty.VectorAgents; import com.microsoft.prompty.model.Agent; +import com.microsoft.prompty.model.ModelInvocationContextSnapshot; +import com.microsoft.prompty.model.ModelInvocationRequest; +import com.microsoft.prompty.model.ModelInvocationResponse; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; /** Grades OpenAI response interpretation against the shared {@code spec/vectors/process} suite. */ @@ -41,4 +49,40 @@ Iterable processVectors() { } return tests; } + + @Test + void portableAssistantMessagesUseResponsesMessageContentFallback() { + Agent agent = new Agent(); + Map response = new LinkedHashMap<>(); + response.put("object", "response"); + response.put("id", "resp_123"); + response.put( + "output", + List.of( + Map.of( + "type", + "message", + "role", + "assistant", + "status", + "completed", + "content", + List.of(Map.of("type", "output_text", "text", "Visible final answer."))))); + + ModelInvocationRequest request = new ModelInvocationRequest(); + request.context = new ModelInvocationContextSnapshot(); + + OpenAIProcessor processor = + new OpenAIProcessor() { + @Override + protected boolean supportsResponsesContinuation() { + return false; + } + }; + ModelInvocationResponse result = processor.processWithContext(agent, response, request); + + assertEquals("Visible final answer.", result.output); + assertEquals(1, result.assistantMessages.size()); + assertEquals("Visible final answer.", Messages.text(result.assistantMessages.get(0))); + } } diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/ProcessorConformanceTests.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/ProcessorConformanceTests.java index 6ea3ac252..71cdd49f8 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/ProcessorConformanceTests.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/ProcessorConformanceTests.java @@ -280,7 +280,35 @@ private static void vector18ProcessorProcessResponsesEmptyOutput() throws Except assertNotNull(actual, "responses_empty_output"); } - private static void vector19ProcessorProcessResponsesStructured() throws Exception { + private static void vector19ProcessorProcessResponsesEmptyOutputTextMessageContentFallback() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"response\":{\"id\":\"resp_empty_output_text_fallback\",\"object\":\"response\",\"created_at\":1700000024,\"model\":\"gpt-5.5-pro\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"msg_empty_output_text_fallback\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"Nested text wins.\",\"annotations\":[]}]}],\"output_text\":\"\",\"usage\":{\"input_tokens\":12,\"output_tokens\":4,\"total_tokens\":16}},\"has_outputs\":false}"); + Object payload = TypraJson.parse(sb.toString()); + java.util.Map input = (java.util.Map) payload; + Agent agent = Agent.load(input.get("agent"), null); + Object response = (Object) input.get("response"); + String provider = agent.model.provider; + Processor impl = ProcessorResolver.resolve(provider, VectorProviders.processor()); + if (impl == null) throw new AssertionError("responses_empty_output_text_message_content_fallback: no Processor attached for " + provider); + Object actual = impl.process(agent, response); + assertNotNull(actual, "responses_empty_output_text_message_content_fallback"); + } + + private static void vector20ProcessorProcessResponsesMessageContentFallback() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"response\":{\"id\":\"resp_content_fallback\",\"object\":\"response\",\"created_at\":1700000023,\"model\":\"gpt-5.5-pro\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"msg_content_fallback\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"Visible final answer.\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":12,\"output_tokens\":4,\"total_tokens\":16}},\"has_outputs\":false}"); + Object payload = TypraJson.parse(sb.toString()); + java.util.Map input = (java.util.Map) payload; + Agent agent = Agent.load(input.get("agent"), null); + Object response = (Object) input.get("response"); + String provider = agent.model.provider; + Processor impl = ProcessorResolver.resolve(provider, VectorProviders.processor()); + if (impl == null) throw new AssertionError("responses_message_content_fallback: no Processor attached for " + provider); + Object actual = impl.process(agent, response); + assertNotNull(actual, "responses_message_content_fallback"); + } + + private static void vector21ProcessorProcessResponsesStructured() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"response\":{\"id\":\"resp_struct1\",\"object\":\"response\",\"created_at\":1700000022,\"model\":\"gpt-4o-mini\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"msg_002\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"annotations\":[]}]}],\"output_text\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"usage\":{\"input_tokens\":20,\"output_tokens\":8,\"total_tokens\":28}},\"has_outputs\":true}"); Object payload = TypraJson.parse(sb.toString()); @@ -294,7 +322,7 @@ private static void vector19ProcessorProcessResponsesStructured() throws Excepti assertNotNull(actual, "responses_structured"); } - private static void vector20ProcessorProcessResponsesText() throws Exception { + private static void vector22ProcessorProcessResponsesText() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"response\":{\"id\":\"resp_abc123\",\"object\":\"response\",\"created_at\":1700000020,\"model\":\"gpt-4o-mini\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"msg_001\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"Hello!\",\"annotations\":[]}]}],\"output_text\":\"Hello!\",\"usage\":{\"input_tokens\":10,\"output_tokens\":2,\"total_tokens\":12}},\"has_outputs\":false}"); Object payload = TypraJson.parse(sb.toString()); @@ -308,7 +336,7 @@ private static void vector20ProcessorProcessResponsesText() throws Exception { assertNotNull(actual, "responses_text"); } - private static void vector21ProcessorProcessResponsesToolCalls() throws Exception { + private static void vector23ProcessorProcessResponsesToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"response\":{\"id\":\"resp_tool1\",\"object\":\"response\",\"created_at\":1700000021,\"model\":\"gpt-4o-mini\",\"status\":\"completed\",\"output\":[{\"type\":\"function_call\",\"id\":\"fc_001\",\"call_id\":\"call_1\",\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"status\":\"completed\"}],\"output_text\":\"\",\"usage\":{\"input_tokens\":25,\"output_tokens\":18,\"total_tokens\":43}},\"has_outputs\":false}"); Object payload = TypraJson.parse(sb.toString()); @@ -322,7 +350,7 @@ private static void vector21ProcessorProcessResponsesToolCalls() throws Exceptio assertNotNull(actual, "responses_tool_calls"); } - private static void vector22ProcessorProcessStreamStreamRefusalIsDeterminate() throws Exception { + private static void vector24ProcessorProcessStreamStreamRefusalIsDeterminate() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"stream\":[{\"kind\":\"provider\",\"value\":{\"choices\":[{\"delta\":{\"refusal\":\"I cannot help with that\"}}]}}],\"events\":[{\"kind\":\"provider\",\"value\":{\"choices\":[{\"delta\":{\"refusal\":\"I cannot help with that\"}}]}}]}"); Object payload = TypraJson.parse(sb.toString()); @@ -357,10 +385,12 @@ public static void run() { failed += runCase("Processor.process:image_url", ProcessorConformanceTests::vector16ProcessorProcessImageUrl); failed += runCase("Processor.processStream:partial_text_then_indeterminate_failure", ProcessorConformanceTests::vector17ProcessorProcessStreamPartialTextThenIndeterminateFailure); failed += runCase("Processor.process:responses_empty_output", ProcessorConformanceTests::vector18ProcessorProcessResponsesEmptyOutput); - failed += runCase("Processor.process:responses_structured", ProcessorConformanceTests::vector19ProcessorProcessResponsesStructured); - failed += runCase("Processor.process:responses_text", ProcessorConformanceTests::vector20ProcessorProcessResponsesText); - failed += runCase("Processor.process:responses_tool_calls", ProcessorConformanceTests::vector21ProcessorProcessResponsesToolCalls); - failed += runCase("Processor.processStream:stream_refusal_is_determinate", ProcessorConformanceTests::vector22ProcessorProcessStreamStreamRefusalIsDeterminate); + failed += runCase("Processor.process:responses_empty_output_text_message_content_fallback", ProcessorConformanceTests::vector19ProcessorProcessResponsesEmptyOutputTextMessageContentFallback); + failed += runCase("Processor.process:responses_message_content_fallback", ProcessorConformanceTests::vector20ProcessorProcessResponsesMessageContentFallback); + failed += runCase("Processor.process:responses_structured", ProcessorConformanceTests::vector21ProcessorProcessResponsesStructured); + failed += runCase("Processor.process:responses_text", ProcessorConformanceTests::vector22ProcessorProcessResponsesText); + failed += runCase("Processor.process:responses_tool_calls", ProcessorConformanceTests::vector23ProcessorProcessResponsesToolCalls); + failed += runCase("Processor.processStream:stream_refusal_is_determinate", ProcessorConformanceTests::vector24ProcessorProcessStreamStreamRefusalIsDeterminate); if (failed > 0) { throw new AssertionError(failed + " @vector conformance failure(s)"); } diff --git a/runtime/python/prompty/prompty/providers/openai/processor.py b/runtime/python/prompty/prompty/providers/openai/processor.py index 1a4525777..e126bbc2f 100644 --- a/runtime/python/prompty/prompty/providers/openai/processor.py +++ b/runtime/python/prompty/prompty/providers/openai/processor.py @@ -255,7 +255,7 @@ def _process_responses_api(response: Any, agent: Agent | None = None) -> Any: # Text content ā€” use output_text convenience field output_text = getattr(response, "output_text", None) - if output_text is not None: + if output_text: if agent is not None and agent.outputs and isinstance(output_text, str): try: return StructuredResult(json.loads(output_text), output_text) diff --git a/runtime/python/prompty/tests/model/test_processor_conformance.py b/runtime/python/prompty/tests/model/test_processor_conformance.py index 072bc207c..1553fd763 100644 --- a/runtime/python/prompty/tests/model/test_processor_conformance.py +++ b/runtime/python/prompty/tests/model/test_processor_conformance.py @@ -920,6 +920,113 @@ def test_responses_empty_output(processor_provider): assert result is not None +def test_responses_empty_output_text_message_content_fallback(processor_provider): + payload = json.loads( + r""" + { + "agent": { + "model": { + "provider": "openai" + } + }, + "apiType": "responses", + "response": { + "id": "resp_empty_output_text_fallback", + "object": "response", + "created_at": 1700000024, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_empty_output_text_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Nested text wins.", + "annotations": [] + } + ] + } + ], + "output_text": "", + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } + }, + "has_outputs": false + } + """, + strict=False, + ) + agent = Agent.load(payload["agent"]) + response = payload["response"] + provider = agent.model.provider + impl = resolve_processor(provider, processor_provider) + assert impl is not None, f"responses_empty_output_text_message_content_fallback: no Processor attached for {provider}" + result = impl.process(agent, response) + if inspect.isawaitable(result): + result = asyncio.run(result) + assert result is not None + + +def test_responses_message_content_fallback(processor_provider): + payload = json.loads( + r""" + { + "agent": { + "model": { + "provider": "openai" + } + }, + "apiType": "responses", + "response": { + "id": "resp_content_fallback", + "object": "response", + "created_at": 1700000023, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_content_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Visible final answer.", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } + }, + "has_outputs": false + } + """, + strict=False, + ) + agent = Agent.load(payload["agent"]) + response = payload["response"] + provider = agent.model.provider + impl = resolve_processor(provider, processor_provider) + assert impl is not None, f"responses_message_content_fallback: no Processor attached for {provider}" + result = impl.process(agent, response) + if inspect.isawaitable(result): + result = asyncio.run(result) + assert result is not None + + def test_responses_structured(processor_provider): payload = json.loads( r""" diff --git a/runtime/rust/prompty-openai/src/processor.rs b/runtime/rust/prompty-openai/src/processor.rs index 15c64833e..1fb167d23 100644 --- a/runtime/rust/prompty-openai/src/processor.rs +++ b/runtime/rust/prompty-openai/src/processor.rs @@ -177,10 +177,7 @@ fn portable_assistant_messages(response: &Value) -> Vec { } return vec![Message::with_text( Role::Assistant, - response - .get("output_text") - .and_then(Value::as_str) - .unwrap_or(""), + extract_responses_output_text(response), )]; } @@ -339,22 +336,43 @@ fn process_responses_api(agent: &Agent, response: &Value) -> Result(output_text) { + if let Ok(parsed) = serde_json::from_str::(&output_text) { return Ok(parsed); } } } - Ok(Value::String(output_text.to_string())) + Ok(Value::String(output_text)) +} + +fn extract_responses_output_text(response: &Value) -> String { + if let Some(output_text) = response.get("output_text").and_then(Value::as_str) { + if !output_text.is_empty() { + return output_text.to_string(); + } + } + + response + .get("output") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("message")) + .filter_map(|item| item.get("content").and_then(Value::as_array)) + .flatten() + .filter(|part| { + matches!( + part.get("type").and_then(Value::as_str), + Some("output_text" | "text") + ) + }) + .filter_map(|part| part.get("text").and_then(Value::as_str)) + .collect::() } // --------------------------------------------------------------------------- @@ -892,6 +910,41 @@ mod tests { ); } + #[test] + fn test_responses_portable_assistant_messages_use_message_content_fallback() { + let agent = make_agent(Value::Null); + let response = json!({ + "object": "response", + "id": "resp_123", + "output": [{ + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{ + "type": "output_text", + "text": "Visible final answer." + }] + }] + }); + + let result = process_invocation_response_with_context( + &agent, + &response, + "openai", + false, + &request(Vec::new()), + ) + .unwrap(); + + assert_eq!(result.output, Some(json!("Visible final answer."))); + let assistant_messages = result.assistant_messages.as_ref().unwrap(); + assert_eq!(assistant_messages.len(), 1); + assert_eq!( + assistant_messages[0].text_content(), + "Visible final answer." + ); + } + #[test] fn test_process_chat_tool_calls() { let agent = make_agent(Value::Null); diff --git a/runtime/rust/prompty-openai/tests/process_vectors.rs b/runtime/rust/prompty-openai/tests/process_vectors.rs index e2c8a3b68..f195d5d4b 100644 --- a/runtime/rust/prompty-openai/tests/process_vectors.rs +++ b/runtime/rust/prompty-openai/tests/process_vectors.rs @@ -152,5 +152,7 @@ process_test!(chat_empty_content); // Responses API vectors ā€” skip if we don't support responses API yet process_test!(responses_text); process_test!(responses_tool_calls); +process_test!(responses_message_content_fallback); +process_test!(responses_empty_output_text_message_content_fallback); process_test!(responses_structured); process_test!(responses_empty_output); diff --git a/runtime/rust/prompty/tests/model/processor_conformance_test.rs b/runtime/rust/prompty/tests/model/processor_conformance_test.rs index 609ac7645..86073e425 100644 --- a/runtime/rust/prompty/tests/model/processor_conformance_test.rs +++ b/runtime/rust/prompty/tests/model/processor_conformance_test.rs @@ -1069,6 +1069,133 @@ async fn test_processor_process_responses_empty_output() { let _ = actual; } +#[tokio::test] +async fn test_processor_process_responses_empty_output_text_message_content_fallback() { + let ctx = LoadContext::default(); + let agent = Agent::from_json( + r####" +{ + "model": { + "provider": "openai" + } +} +"####, + &ctx, + ) + .expect("agent parses"); + let response: serde_json::Value = serde_json::from_str( + r####" +{ + "id": "resp_empty_output_text_fallback", + "object": "response", + "created_at": 1700000024, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_empty_output_text_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Nested text wins.", + "annotations": [] + } + ] + } + ], + "output_text": "", + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } +} +"####, + ) + .expect("response parses"); + let kind = agent + .model + .get("provider") + .and_then(|v| v.as_str()) + .expect("discriminator present"); + let provider = vector_adapters::processor_provider(); + let seam_impl = processor_resolver::resolve(kind, &provider).unwrap_or_else(|| { + panic!( + "responses_empty_output_text_message_content_fallback: no Processor attached for {kind}" + ) + }); + let actual = seam_impl + .process(&agent, &response) + .await + .expect("seam invocation"); + let _ = actual; +} + +#[tokio::test] +async fn test_processor_process_responses_message_content_fallback() { + let ctx = LoadContext::default(); + let agent = Agent::from_json( + r####" +{ + "model": { + "provider": "openai" + } +} +"####, + &ctx, + ) + .expect("agent parses"); + let response: serde_json::Value = serde_json::from_str( + r####" +{ + "id": "resp_content_fallback", + "object": "response", + "created_at": 1700000023, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_content_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Visible final answer.", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } +} +"####, + ) + .expect("response parses"); + let kind = agent + .model + .get("provider") + .and_then(|v| v.as_str()) + .expect("discriminator present"); + let provider = vector_adapters::processor_provider(); + let seam_impl = processor_resolver::resolve(kind, &provider).unwrap_or_else(|| { + panic!("responses_message_content_fallback: no Processor attached for {kind}") + }); + let actual = seam_impl + .process(&agent, &response) + .await + .expect("seam invocation"); + let _ = actual; +} + #[tokio::test] async fn test_processor_process_responses_structured() { let ctx = LoadContext::default(); diff --git a/runtime/swift/prompty-model/Sources/PromptyModel/PipelineRuntime.swift b/runtime/swift/prompty-model/Sources/PromptyModel/PipelineRuntime.swift index a2c920bce..4fe7f6db1 100644 --- a/runtime/swift/prompty-model/Sources/PromptyModel/PipelineRuntime.swift +++ b/runtime/swift/prompty-model/Sources/PromptyModel/PipelineRuntime.swift @@ -306,7 +306,24 @@ private func processOpenAIResponses(_ resp: [String: Any], hasOutputs: Bool) -> toolCalls.append(["id": item["call_id"] ?? NSNull(), "name": item["name"] ?? NSNull(), "arguments": item["arguments"] ?? NSNull()]) } if !toolCalls.isEmpty { return toolCalls } - return maybeParseStructured(resp["output_text"] as? String ?? "", hasOutputs) + return maybeParseStructured(openAIResponsesOutputText(resp), hasOutputs) +} + +private func openAIResponsesOutputText(_ resp: [String: Any]) -> String { + if let outputText = resp["output_text"] as? String, !outputText.isEmpty { + return outputText + } + + var text = "" + for item in resp["output"] as? [[String: Any]] ?? [] where (item["type"] as? String) == "message" { + for part in item["content"] as? [[String: Any]] ?? [] { + let partType = part["type"] as? String + if partType == "output_text" || partType == "text" { + text += part["text"] as? String ?? "" + } + } + } + return text } private func processOpenAIEmbedding(_ resp: [String: Any]) -> Any { diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/PipelineRuntimeTests.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/PipelineRuntimeTests.swift new file mode 100644 index 000000000..d15133c35 --- /dev/null +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/PipelineRuntimeTests.swift @@ -0,0 +1,31 @@ +import XCTest +@testable import PromptyModel + +final class PipelineRuntimeTests: XCTestCase { + func testOpenAIResponsesEmptyOutputTextFallsBackToMessageContent() throws { + let response: [String: Any] = [ + "object": "response", + "output_text": "", + "output": [ + [ + "type": "message", + "content": [ + [ + "type": "output_text", + "text": "Nested text wins.", + ] + ], + ] + ], + ] + + let result = processResponse( + provider: "openai", + apiType: "responses", + response: response, + hasOutputs: false + ) + + XCTAssertEqual(result as? String, "Nested text wins.") + } +} diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/ProcessorConformanceTests.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/ProcessorConformanceTests.swift index 429b5de14..8f7128406 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/ProcessorConformanceTests.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/ProcessorConformanceTests.swift @@ -364,7 +364,43 @@ final class ProcessorConformanceTests: XCTestCase { XCTAssertNotNil(actual) } - func testVector19ProcessorProcessResponsesStructured() async throws { + func testVector19ProcessorProcessResponsesEmptyOutputTextMessageContentFallback() async throws { + guard let inputData = "{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"response\":{\"id\":\"resp_empty_output_text_fallback\",\"object\":\"response\",\"created_at\":1700000024,\"model\":\"gpt-5.5-pro\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"msg_empty_output_text_fallback\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"Nested text wins.\",\"annotations\":[]}]}],\"output_text\":\"\",\"usage\":{\"input_tokens\":12,\"output_tokens\":4,\"total_tokens\":16}},\"has_outputs\":false}".data(using: .utf8), + let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector input") + return + } + let agent = try Agent.load(input["agent"]!) + let response = input["response"] as! Any + let providerKind = try (agent.model!.save())["provider"] as! String + guard let impl = ProcessorResolver.resolve(provider: providerKind, registry: provider()) else { + XCTFail("responses_empty_output_text_message_content_fallback: no Processor attached for " + providerKind) + return + } + let actual = try await impl.process(agent: agent, response: response) + XCTAssertNotNil(actual) + } + + func testVector20ProcessorProcessResponsesMessageContentFallback() async throws { + guard let inputData = "{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"response\":{\"id\":\"resp_content_fallback\",\"object\":\"response\",\"created_at\":1700000023,\"model\":\"gpt-5.5-pro\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"msg_content_fallback\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"Visible final answer.\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":12,\"output_tokens\":4,\"total_tokens\":16}},\"has_outputs\":false}".data(using: .utf8), + let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector input") + return + } + let agent = try Agent.load(input["agent"]!) + let response = input["response"] as! Any + let providerKind = try (agent.model!.save())["provider"] as! String + guard let impl = ProcessorResolver.resolve(provider: providerKind, registry: provider()) else { + XCTFail("responses_message_content_fallback: no Processor attached for " + providerKind) + return + } + let actual = try await impl.process(agent: agent, response: response) + XCTAssertNotNil(actual) + } + + func testVector21ProcessorProcessResponsesStructured() async throws { guard let inputData = "{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"response\":{\"id\":\"resp_struct1\",\"object\":\"response\",\"created_at\":1700000022,\"model\":\"gpt-4o-mini\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"msg_002\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"annotations\":[]}]}],\"output_text\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"usage\":{\"input_tokens\":20,\"output_tokens\":8,\"total_tokens\":28}},\"has_outputs\":true}".data(using: .utf8), let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { @@ -382,7 +418,7 @@ final class ProcessorConformanceTests: XCTestCase { XCTAssertNotNil(actual) } - func testVector20ProcessorProcessResponsesText() async throws { + func testVector22ProcessorProcessResponsesText() async throws { guard let inputData = "{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"response\":{\"id\":\"resp_abc123\",\"object\":\"response\",\"created_at\":1700000020,\"model\":\"gpt-4o-mini\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"msg_001\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"Hello!\",\"annotations\":[]}]}],\"output_text\":\"Hello!\",\"usage\":{\"input_tokens\":10,\"output_tokens\":2,\"total_tokens\":12}},\"has_outputs\":false}".data(using: .utf8), let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { @@ -400,7 +436,7 @@ final class ProcessorConformanceTests: XCTestCase { XCTAssertNotNil(actual) } - func testVector21ProcessorProcessResponsesToolCalls() async throws { + func testVector23ProcessorProcessResponsesToolCalls() async throws { guard let inputData = "{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"response\":{\"id\":\"resp_tool1\",\"object\":\"response\",\"created_at\":1700000021,\"model\":\"gpt-4o-mini\",\"status\":\"completed\",\"output\":[{\"type\":\"function_call\",\"id\":\"fc_001\",\"call_id\":\"call_1\",\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"status\":\"completed\"}],\"output_text\":\"\",\"usage\":{\"input_tokens\":25,\"output_tokens\":18,\"total_tokens\":43}},\"has_outputs\":false}".data(using: .utf8), let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { @@ -418,7 +454,7 @@ final class ProcessorConformanceTests: XCTestCase { XCTAssertNotNil(actual) } - func testVector22ProcessorProcessStreamStreamRefusalIsDeterminate() async throws { + func testVector24ProcessorProcessStreamStreamRefusalIsDeterminate() async throws { guard let inputData = "{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"stream\":[{\"kind\":\"provider\",\"value\":{\"choices\":[{\"delta\":{\"refusal\":\"I cannot help with that\"}}]}}],\"events\":[{\"kind\":\"provider\",\"value\":{\"choices\":[{\"delta\":{\"refusal\":\"I cannot help with that\"}}]}}]}".data(using: .utf8), let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { diff --git a/runtime/typescript/packages/core/tests/model/processor.conformance.test.ts b/runtime/typescript/packages/core/tests/model/processor.conformance.test.ts index c435ac6ed..f0fddd4ff 100644 --- a/runtime/typescript/packages/core/tests/model/processor.conformance.test.ts +++ b/runtime/typescript/packages/core/tests/model/processor.conformance.test.ts @@ -245,6 +245,30 @@ describe("Processor @vector conformance", () => { const result = await impl!.process(agent, response); expect(result).toBeDefined(); }); + it("responses_empty_output_text_message_content_fallback", async () => { + const payload = JSON.parse( + '{"agent":{"model":{"provider":"openai"}},"apiType":"responses","response":{"id":"resp_empty_output_text_fallback","object":"response","created_at":1700000024,"model":"gpt-5.5-pro","status":"completed","output":[{"type":"message","id":"msg_empty_output_text_fallback","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Nested text wins.","annotations":[]}]}],"output_text":"","usage":{"input_tokens":12,"output_tokens":4,"total_tokens":16}},"has_outputs":false}', + ) as Record; + const agent = Agent.load(payload["agent"] as Record); + const response = payload["response"] as unknown; + const provider = agent.model.provider; + const impl = resolveProcessor(provider, processorProvider); + expect(impl).not.toBeNull(); + const result = await impl!.process(agent, response); + expect(result).toBeDefined(); + }); + it("responses_message_content_fallback", async () => { + const payload = JSON.parse( + '{"agent":{"model":{"provider":"openai"}},"apiType":"responses","response":{"id":"resp_content_fallback","object":"response","created_at":1700000023,"model":"gpt-5.5-pro","status":"completed","output":[{"type":"message","id":"msg_content_fallback","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Visible final answer.","annotations":[]}]}],"usage":{"input_tokens":12,"output_tokens":4,"total_tokens":16}},"has_outputs":false}', + ) as Record; + const agent = Agent.load(payload["agent"] as Record); + const response = payload["response"] as unknown; + const provider = agent.model.provider; + const impl = resolveProcessor(provider, processorProvider); + expect(impl).not.toBeNull(); + const result = await impl!.process(agent, response); + expect(result).toBeDefined(); + }); it("responses_structured", async () => { const payload = JSON.parse( '{"agent":{"model":{"provider":"openai"}},"apiType":"responses","response":{"id":"resp_struct1","object":"response","created_at":1700000022,"model":"gpt-4o-mini","status":"completed","output":[{"type":"message","id":"msg_002","status":"completed","role":"assistant","content":[{"type":"output_text","text":"{\\"city\\":\\"Paris\\"}","annotations":[]}]}],"output_text":"{\\"city\\":\\"Paris\\"}","usage":{"input_tokens":20,"output_tokens":8,"total_tokens":28}},"has_outputs":true}', diff --git a/runtime/typescript/packages/openai/src/processor.ts b/runtime/typescript/packages/openai/src/processor.ts index 9ea9310a8..5c4f49c1f 100644 --- a/runtime/typescript/packages/openai/src/processor.ts +++ b/runtime/typescript/packages/openai/src/processor.ts @@ -238,7 +238,7 @@ function processResponsesApi( // Text content ā€” use output_text convenience field const outputText = response.output_text as string | undefined; - if (outputText !== undefined) { + if (outputText) { // Structured output ā€” JSON parse when outputs schema exists if (agent.outputs && agent.outputs.length > 0) { try { diff --git a/runtime/typescript/packages/openai/tests/kinds.test.ts b/runtime/typescript/packages/openai/tests/kinds.test.ts index 623bbd597..6f1069205 100644 --- a/runtime/typescript/packages/openai/tests/kinds.test.ts +++ b/runtime/typescript/packages/openai/tests/kinds.test.ts @@ -812,14 +812,24 @@ describe("processResponse Responses API", () => { expect(processResponse(agent, response)).toEqual({ summary: "test", score: 5 }); }); - it("falls back to message content when output_text is absent", () => { + it("falls back to message content when output_text is absent or empty", () => { const agent = makeAgent({ apiType: "responses" }); - const response = { - object: "response", - output: [ - { type: "message", content: [{ type: "output_text", text: "Fallback text" }] }, - ], - }; - expect(processResponse(agent, response)).toBe("Fallback text"); + for (const response of [ + { + object: "response", + output: [ + { type: "message", content: [{ type: "output_text", text: "Fallback text" }] }, + ], + }, + { + object: "response", + output: [ + { type: "message", content: [{ type: "output_text", text: "Fallback text" }] }, + ], + output_text: "", + }, + ]) { + expect(processResponse(agent, response)).toBe("Fallback text"); + } }); }); diff --git a/schema/model/conformance/vectors/process.tsp b/schema/model/conformance/vectors/process.tsp index a61ccc150..f416cd960 100644 --- a/schema/model/conformance/vectors/process.tsp +++ b/schema/model/conformance/vectors/process.tsp @@ -548,6 +548,87 @@ const ProcessVectors = """ ] } }, + { + "name": "responses_message_content_fallback", + "description": "Responses API completed message without output_text ā€” extracts output[].content[].output_text", + "stage": "process", + "input": { + "agent": { "model": { "provider": "openai" } }, + "apiType": "responses", + "response": { + "id": "resp_content_fallback", + "object": "response", + "created_at": 1700000023, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_content_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Visible final answer.", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } + }, + "has_outputs": false + }, + "expected": { + "result": "Visible final answer." + } + }, + { + "name": "responses_empty_output_text_message_content_fallback", + "description": "Responses API completed message with empty output_text ā€” falls back to output[].content[].output_text", + "stage": "process", + "input": { + "agent": { "model": { "provider": "openai" } }, + "apiType": "responses", + "response": { + "id": "resp_empty_output_text_fallback", + "object": "response", + "created_at": 1700000024, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_empty_output_text_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Nested text wins.", + "annotations": [] + } + ] + } + ], + "output_text": "", + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } + }, + "has_outputs": false + }, + "expected": { + "result": "Nested text wins." + } + }, { "name": "responses_structured", "description": "Responses API with structured output (has_outputs=true) ā€” JSON-parses output_text into object", diff --git a/schema/tsp-output/.typra-generated/vectors.json b/schema/tsp-output/.typra-generated/vectors.json index d2e257ae5..23feeae50 100644 --- a/schema/tsp-output/.typra-generated/vectors.json +++ b/schema/tsp-output/.typra-generated/vectors.json @@ -5716,6 +5716,191 @@ "operation": "process" } }, + { + "contract": "Processor", + "namespace": "Prompty.Operations.Pipeline", + "group": "", + "operation": "process", + "params": { + "agent": "Agent", + "response": "unknown" + }, + "returns": "unknown", + "sync": false, + "dispatch": { + "discriminator": { + "model": "Model", + "field": "provider" + }, + "path": "agent.model.provider", + "decl": { + "discriminatorField": "provider", + "variants": [ + { + "value": "openai", + "typeName": { + "namespace": "Prompty.Contracts.Models", + "name": "OpenAIModel" + } + }, + { + "value": "azure", + "typeName": { + "namespace": "Prompty.Contracts.Models", + "name": "AzureModel" + } + } + ], + "defaultVariant": { + "typeName": { + "namespace": "Prompty.Contracts.Models", + "name": "CustomModel" + }, + "isSelfReference": false + }, + "isAbstract": false, + "isClosed": false + } + }, + "vector": { + "name": "responses_empty_output_text_message_content_fallback", + "description": "Responses API completed message with empty output_text ā€” falls back to output[].content[].output_text", + "stage": "process", + "input": { + "agent": { + "model": { + "provider": "openai" + } + }, + "apiType": "responses", + "response": { + "id": "resp_empty_output_text_fallback", + "object": "response", + "created_at": 1700000024, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_empty_output_text_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Nested text wins.", + "annotations": [] + } + ] + } + ], + "output_text": "", + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } + }, + "has_outputs": false + }, + "expected": { + "result": "Nested text wins." + }, + "operation": "process" + } + }, + { + "contract": "Processor", + "namespace": "Prompty.Operations.Pipeline", + "group": "", + "operation": "process", + "params": { + "agent": "Agent", + "response": "unknown" + }, + "returns": "unknown", + "sync": false, + "dispatch": { + "discriminator": { + "model": "Model", + "field": "provider" + }, + "path": "agent.model.provider", + "decl": { + "discriminatorField": "provider", + "variants": [ + { + "value": "openai", + "typeName": { + "namespace": "Prompty.Contracts.Models", + "name": "OpenAIModel" + } + }, + { + "value": "azure", + "typeName": { + "namespace": "Prompty.Contracts.Models", + "name": "AzureModel" + } + } + ], + "defaultVariant": { + "typeName": { + "namespace": "Prompty.Contracts.Models", + "name": "CustomModel" + }, + "isSelfReference": false + }, + "isAbstract": false, + "isClosed": false + } + }, + "vector": { + "name": "responses_message_content_fallback", + "description": "Responses API completed message without output_text ā€” extracts output[].content[].output_text", + "stage": "process", + "input": { + "agent": { + "model": { + "provider": "openai" + } + }, + "apiType": "responses", + "response": { + "id": "resp_content_fallback", + "object": "response", + "created_at": 1700000023, + "model": "gpt-5.5-pro", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_content_fallback", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Visible final answer.", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 12, + "output_tokens": 4, + "total_tokens": 16 + } + }, + "has_outputs": false + }, + "expected": { + "result": "Visible final answer." + }, + "operation": "process" + } + }, { "contract": "Processor", "namespace": "Prompty.Operations.Pipeline", From f583c07f8f3322f0c212aeda49590381b8c15c79 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Thu, 15 Oct 2026 21:49:29 -0700 Subject: [PATCH 3/3] test(python): format generated processor conformance Format the generated Responses fallback conformance assertion so CI ruff format passes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 497512b4-8e78-461b-9e96-8e9af3fdb03c --- .../python/prompty/tests/model/test_processor_conformance.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/python/prompty/tests/model/test_processor_conformance.py b/runtime/python/prompty/tests/model/test_processor_conformance.py index 1553fd763..5dba5c1b2 100644 --- a/runtime/python/prompty/tests/model/test_processor_conformance.py +++ b/runtime/python/prompty/tests/model/test_processor_conformance.py @@ -967,7 +967,9 @@ def test_responses_empty_output_text_message_content_fallback(processor_provider response = payload["response"] provider = agent.model.provider impl = resolve_processor(provider, processor_provider) - assert impl is not None, f"responses_empty_output_text_message_content_fallback: no Processor attached for {provider}" + assert impl is not None, ( + f"responses_empty_output_text_message_content_fallback: no Processor attached for {provider}" + ) result = impl.process(agent, response) if inspect.isawaitable(result): result = asyncio.run(result)