From d8cb188f92edb3d3a934f1d2a884264926d4f49d Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Fri, 4 Sep 2026 16:03:01 -0700 Subject: [PATCH 1/3] feat(model): service-drill conformance vector on the callable seam Author the first prompty "service drill" -- a conformance vector that crosses the real Executor transport seam (live when creds exist, cassette-replay when absent) and asserts three planes: transport, wire, and semantic. The semantic plane IS a conformance vector over the Executor path. Spike finding: the existing @typra/emitter (2.1.7) carries a stage:"drill" vector plus its unknown `exchange` block through unchanged (unknown-key passthrough), into vectors.json and all 7 runtime harnesses -- zero emitter change needed to author or deliver a drill. - schema: add LiveChatConformance seam + drill.tsp vector; regen 7 runtimes. - python: drill adapter (base-URL redirect -> stdlib cassette-replay server, 3-plane assert + live/replay parity), cassette + recorder tooling. - typescript: mirror adapter + Node cassette-replay server + cassette copy. Drill agent built from @prompty/core (dist) so the executor's instanceof ApiKeyConnection matches across the dual-core test seam. - go/rust/swift/csharp/java: operation-level waivers so the drill vector skips cleanly where no adapter is registered (Python/TS-only for the spike). The requires guard is all-or-nothing, so the drill vector omits it and the adapter self-gates the live plane on credential presence while always running replay. Cassette bytes, SDK binding, and cred gating stay consumer-owned roster -- by design not modeled in TypeSpec. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 442651bd-34df-4b59-8e2f-d7093fa855a8 --- .../Model/VectorAdapters.cs | 5 +- .../Model/VectorConformanceTests.cs | 243 ++++++----- .../Prompty.Core/Model/LiveChatConformance.cs | 31 ++ .../go/prompty/model/live_chat_conformance.go | 17 + .../prompty/model/vector_conformance_test.go | 242 ++++++----- .../go/prompty/model/vectorbridge/bridge.go | 21 + .../prompty/vectoradapters/vectoradapters.go | 4 +- .../prompty/model/LiveChatConformance.java | 12 + .../prompty/model/VectorAdapters.java | 6 +- .../prompty/model/VectorConformanceTests.java | 384 +++++++++--------- .../prompty/model/_LiveChatConformance.py | 28 ++ .../python/prompty/prompty/model/__init__.py | 2 + .../prompty/tests/model/_drill_replay.py | 97 +++++ .../model/cassettes/openai_chat_drill.json | 44 ++ .../prompty/tests/model/record_cassette.py | 107 +++++ .../model/test_capability_guard_reference.py | 17 +- .../tests/model/test_vector_conformance.py | 241 ++++++----- .../prompty/tests/model/vector_adapters.py | 169 ++++++-- .../src/model/live_chat_conformance.rs | 21 + runtime/rust/prompty/src/model/mod.rs | 3 + .../prompty/tests/model/vector_adapters.rs | 5 +- .../tests/model/vector_conformance_test.rs | 195 ++++----- .../PromptyModel/live_chat_conformance.swift | 8 + .../PromptyModelTests/VectorAdapters.swift | 5 +- .../VectorConformanceTests.swift | 198 ++++----- .../packages/core/src/model/index.ts | 1 + .../core/src/model/live-chat-conformance.ts | 15 + .../model/cassettes/openai_chat_drill.json | 44 ++ .../packages/core/tests/model/drill-replay.ts | 94 +++++ .../core/tests/model/vector-adapters.ts | 260 ++++++++++++ .../tests/model/vector-conformance.test.ts | 6 + .../seams/live-chat-conformance.tsp | 24 ++ schema/model/conformance/vectors/drill.tsp | 55 +++ schema/model/main.tsp | 8 + .../.typra-generated/export-surfaces.json | 232 +++++++++++ .../.typra-generated/hydration-seams.json | 64 +++ .../tsp-output/.typra-generated/manifest.json | 40 ++ .../tsp-output/.typra-generated/vectors.json | 57 +++ .../docs/reference/LiveChatConformance.md | 40 ++ 39 files changed, 2346 insertions(+), 699 deletions(-) create mode 100644 runtime/csharp/Prompty.Core/Model/LiveChatConformance.cs create mode 100644 runtime/go/prompty/model/live_chat_conformance.go create mode 100644 runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LiveChatConformance.java create mode 100644 runtime/python/prompty/prompty/model/_LiveChatConformance.py create mode 100644 runtime/python/prompty/tests/model/_drill_replay.py create mode 100644 runtime/python/prompty/tests/model/cassettes/openai_chat_drill.json create mode 100644 runtime/python/prompty/tests/model/record_cassette.py create mode 100644 runtime/rust/prompty/src/model/live_chat_conformance.rs create mode 100644 runtime/swift/prompty-model/Sources/PromptyModel/live_chat_conformance.swift create mode 100644 runtime/typescript/packages/core/src/model/live-chat-conformance.ts create mode 100644 runtime/typescript/packages/core/tests/model/cassettes/openai_chat_drill.json create mode 100644 runtime/typescript/packages/core/tests/model/drill-replay.ts create mode 100644 schema/model/conformance/seams/live-chat-conformance.tsp create mode 100644 schema/model/conformance/vectors/drill.tsp create mode 100644 web/src/content/docs/reference/LiveChatConformance.md diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs index 422839d89..889793989 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs @@ -74,7 +74,10 @@ public static partial class VectorAdapters ["WireConformance.toRequest"] = new(WireInvoke, AlignNormalize), }; - public static IReadOnlyDictionary Waivers() => new Dictionary(); + public static IReadOnlyDictionary Waivers() => new Dictionary + { + ["LiveChatConformance.complete"] = "drill: consumer-owned transport-seam probe (Python/TS only)", + }; public static JsonNode? Doubles() => null; diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs index 603b74058..655221cdc 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs @@ -570,7 +570,62 @@ public async Task Vector15DiscoveryConformanceMapModelOpenaiModelFinetuneNoOwner } [Fact] - public async Task Vector16LoadConformanceLoadBasicLoad() + public async Task Vector16LiveChatConformanceCompleteOpenaiChatDrill() + { + string vectorJson = """ +{ + "name": "openai_chat_drill", + "stage": "drill", + "provider": "openai", + "input": { + "provider": "openai", + "model": "gpt-4o-mini", + "apiKey": { + "$env": "OPENAI_API_KEY" + }, + "endpoint": { + "$env": "OPENAI_BASE_URL" + }, + "messages": [ + { + "role": "user", + "content": "Say hello in exactly one word." + } + ], + "options": { + "temperature": 0, + "maxOutputTokens": 16 + } + }, + "exchange": { + "transport": { + "baseUrl": { + "$env": "OPENAI_BASE_URL" + } + }, + "cassette": { + "$json": "cassettes/openai_chat_drill.json" + }, + "planes": [ + "transport", + "wire", + "semantic" + ] + }, + "expected": { + "role": "assistant", + "contentNonEmpty": true, + "finishReasonInEnum": true + }, + "operation": "complete" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LiveChatConformance", "complete", vector, false, Seam()); + } + + [Fact] + public async Task Vector17LoadConformanceLoadBasicLoad() { string vectorJson = """ { @@ -641,7 +696,7 @@ public async Task Vector16LoadConformanceLoadBasicLoad() } [Fact] - public async Task Vector17LoadConformanceLoadConnectionTypesLoad() + public async Task Vector18LoadConformanceLoadConnectionTypesLoad() { string vectorJson = """ { @@ -677,7 +732,7 @@ public async Task Vector17LoadConformanceLoadConnectionTypesLoad() } [Fact] - public async Task Vector18LoadConformanceLoadEmbeddingLoad() + public async Task Vector19LoadConformanceLoadEmbeddingLoad() { string vectorJson = """ { @@ -720,7 +775,7 @@ public async Task Vector18LoadConformanceLoadEmbeddingLoad() } [Fact] - public async Task Vector19LoadConformanceLoadEmptyFrontmatterBodyOnly() + public async Task Vector20LoadConformanceLoadEmptyFrontmatterBodyOnly() { string vectorJson = """ { @@ -747,7 +802,7 @@ public async Task Vector19LoadConformanceLoadEmptyFrontmatterBodyOnly() } [Fact] - public async Task Vector20LoadConformanceLoadEnvDefault() + public async Task Vector21LoadConformanceLoadEnvDefault() { string vectorJson = """ { @@ -783,7 +838,7 @@ public async Task Vector20LoadConformanceLoadEnvDefault() } [Fact] - public async Task Vector21LoadConformanceLoadEnvMissingError() + public async Task Vector22LoadConformanceLoadEnvMissingError() { string vectorJson = """ { @@ -814,7 +869,7 @@ public async Task Vector21LoadConformanceLoadEnvMissingError() } [Fact] - public async Task Vector22LoadConformanceLoadEnvResolution() + public async Task Vector23LoadConformanceLoadEnvResolution() { string vectorJson = """ { @@ -852,7 +907,7 @@ public async Task Vector22LoadConformanceLoadEnvResolution() } [Fact] - public async Task Vector23LoadConformanceLoadFileReferenceInTreeAllowed() + public async Task Vector24LoadConformanceLoadFileReferenceInTreeAllowed() { string vectorJson = """ { @@ -892,7 +947,7 @@ public async Task Vector23LoadConformanceLoadFileReferenceInTreeAllowed() } [Fact] - public async Task Vector24LoadConformanceLoadFileReferenceTraversalRejected() + public async Task Vector25LoadConformanceLoadFileReferenceTraversalRejected() { string vectorJson = """ { @@ -927,7 +982,7 @@ public async Task Vector24LoadConformanceLoadFileReferenceTraversalRejected() } [Fact] - public async Task Vector25LoadConformanceLoadFileResolution() + public async Task Vector26LoadConformanceLoadFileResolution() { string vectorJson = """ { @@ -969,7 +1024,7 @@ public async Task Vector25LoadConformanceLoadFileResolution() } [Fact] - public async Task Vector26LoadConformanceLoadImageApitypeLoad() + public async Task Vector27LoadConformanceLoadImageApitypeLoad() { string vectorJson = """ { @@ -1015,7 +1070,7 @@ public async Task Vector26LoadConformanceLoadImageApitypeLoad() } [Fact] - public async Task Vector27LoadConformanceLoadInputScalarShorthand() + public async Task Vector28LoadConformanceLoadInputScalarShorthand() { string vectorJson = """ { @@ -1062,7 +1117,7 @@ public async Task Vector27LoadConformanceLoadInputScalarShorthand() } [Fact] - public async Task Vector28LoadConformanceLoadInputValidationDefaultFill() + public async Task Vector29LoadConformanceLoadInputValidationDefaultFill() { string vectorJson = """ { @@ -1096,7 +1151,7 @@ public async Task Vector28LoadConformanceLoadInputValidationDefaultFill() } [Fact] - public async Task Vector29LoadConformanceLoadInputValidationExampleNotUsed() + public async Task Vector30LoadConformanceLoadInputValidationExampleNotUsed() { string vectorJson = """ { @@ -1129,7 +1184,7 @@ public async Task Vector29LoadConformanceLoadInputValidationExampleNotUsed() } [Fact] - public async Task Vector30LoadConformanceLoadInputValidationOptionalOmit() + public async Task Vector31LoadConformanceLoadInputValidationOptionalOmit() { string vectorJson = """ { @@ -1161,7 +1216,7 @@ public async Task Vector30LoadConformanceLoadInputValidationOptionalOmit() } [Fact] - public async Task Vector31LoadConformanceLoadInputValidationRequired() + public async Task Vector32LoadConformanceLoadInputValidationRequired() { string vectorJson = """ { @@ -1194,7 +1249,7 @@ public async Task Vector31LoadConformanceLoadInputValidationRequired() } [Fact] - public async Task Vector32LoadConformanceLoadInstructionsFromBody() + public async Task Vector33LoadConformanceLoadInstructionsFromBody() { string vectorJson = """ { @@ -1219,7 +1274,7 @@ public async Task Vector32LoadConformanceLoadInstructionsFromBody() } [Fact] - public async Task Vector33LoadConformanceLoadInvalidFrontmatterError() + public async Task Vector34LoadConformanceLoadInvalidFrontmatterError() { string vectorJson = """ { @@ -1240,7 +1295,7 @@ public async Task Vector33LoadConformanceLoadInvalidFrontmatterError() } [Fact] - public async Task Vector34LoadConformanceLoadKindAlwaysPrompt() + public async Task Vector35LoadConformanceLoadKindAlwaysPrompt() { string vectorJson = """ { @@ -1261,7 +1316,7 @@ public async Task Vector34LoadConformanceLoadKindAlwaysPrompt() } [Fact] - public async Task Vector35LoadConformanceLoadMinimalLoad() + public async Task Vector36LoadConformanceLoadMinimalLoad() { string vectorJson = """ { @@ -1290,7 +1345,7 @@ public async Task Vector35LoadConformanceLoadMinimalLoad() } [Fact] - public async Task Vector36LoadConformanceLoadMissingFileError() + public async Task Vector37LoadConformanceLoadMissingFileError() { string vectorJson = """ { @@ -1311,7 +1366,7 @@ public async Task Vector36LoadConformanceLoadMissingFileError() } [Fact] - public async Task Vector37LoadConformanceLoadModelShorthand() + public async Task Vector38LoadConformanceLoadModelShorthand() { string vectorJson = """ { @@ -1339,7 +1394,7 @@ public async Task Vector37LoadConformanceLoadModelShorthand() } [Fact] - public async Task Vector38LoadConformanceLoadStructuredOutputsLoad() + public async Task Vector39LoadConformanceLoadStructuredOutputsLoad() { string vectorJson = """ { @@ -1407,7 +1462,7 @@ public async Task Vector38LoadConformanceLoadStructuredOutputsLoad() } [Fact] - public async Task Vector39LoadConformanceLoadTemplateStringInvalid() + public async Task Vector40LoadConformanceLoadTemplateStringInvalid() { string vectorJson = """ { @@ -1434,7 +1489,7 @@ public async Task Vector39LoadConformanceLoadTemplateStringInvalid() } [Fact] - public async Task Vector40LoadConformanceLoadToolsCustomLoad() + public async Task Vector41LoadConformanceLoadToolsCustomLoad() { string vectorJson = """ { @@ -1473,7 +1528,7 @@ public async Task Vector40LoadConformanceLoadToolsCustomLoad() } [Fact] - public async Task Vector41LoadConformanceLoadToolsFunctionLoad() + public async Task Vector42LoadConformanceLoadToolsFunctionLoad() { string vectorJson = """ { @@ -1541,7 +1596,7 @@ public async Task Vector41LoadConformanceLoadToolsFunctionLoad() } [Fact] - public async Task Vector42LoadConformanceLoadToolsMcpLoad() + public async Task Vector43LoadConformanceLoadToolsMcpLoad() { string vectorJson = """ { @@ -1582,7 +1637,7 @@ public async Task Vector42LoadConformanceLoadToolsMcpLoad() } [Fact] - public async Task Vector43LoadConformanceLoadToolsOpenapiLoad() + public async Task Vector44LoadConformanceLoadToolsOpenapiLoad() { string vectorJson = """ { @@ -1623,7 +1678,7 @@ public async Task Vector43LoadConformanceLoadToolsOpenapiLoad() } [Fact] - public async Task Vector44TurnConformanceReplayMaxIterations() + public async Task Vector45TurnConformanceReplayMaxIterations() { string vectorJson = """ { @@ -1660,7 +1715,7 @@ public async Task Vector44TurnConformanceReplayMaxIterations() } [Fact] - public async Task Vector45TurnConformanceReplayNoTool() + public async Task Vector46TurnConformanceReplayNoTool() { string vectorJson = """ { @@ -1693,7 +1748,7 @@ public async Task Vector45TurnConformanceReplayNoTool() } [Fact] - public async Task Vector46TurnConformanceReplayPermissionDenied() + public async Task Vector47TurnConformanceReplayPermissionDenied() { string vectorJson = """ { @@ -1729,7 +1784,7 @@ public async Task Vector46TurnConformanceReplayPermissionDenied() } [Fact] - public async Task Vector47TurnConformanceReplayToolFailure() + public async Task Vector48TurnConformanceReplayToolFailure() { string vectorJson = """ { @@ -1767,7 +1822,7 @@ public async Task Vector47TurnConformanceReplayToolFailure() } [Fact] - public async Task Vector48TurnConformanceReplayToolSuccess() + public async Task Vector49TurnConformanceReplayToolSuccess() { string vectorJson = """ { @@ -1805,7 +1860,7 @@ public async Task Vector48TurnConformanceReplayToolSuccess() } [Fact] - public async Task Vector49TurnConformanceRunAssistantToolCallsMetadata() + public async Task Vector50TurnConformanceRunAssistantToolCallsMetadata() { string vectorJson = """ { @@ -1959,7 +2014,7 @@ public async Task Vector49TurnConformanceRunAssistantToolCallsMetadata() } [Fact] - public async Task Vector50TurnConformanceRunAsyncToolFunction() + public async Task Vector51TurnConformanceRunAsyncToolFunction() { string vectorJson = """ { @@ -2071,7 +2126,7 @@ public async Task Vector50TurnConformanceRunAsyncToolFunction() } [Fact] - public async Task Vector51TurnConformanceRunBindingsInjected() + public async Task Vector52TurnConformanceRunBindingsInjected() { string vectorJson = """ { @@ -2203,7 +2258,7 @@ public async Task Vector51TurnConformanceRunBindingsInjected() } [Fact] - public async Task Vector52TurnConformanceRunCancellationBeforeLlm() + public async Task Vector53TurnConformanceRunCancellationBeforeLlm() { string vectorJson = """ { @@ -2266,7 +2321,7 @@ public async Task Vector52TurnConformanceRunCancellationBeforeLlm() } [Fact] - public async Task Vector53TurnConformanceRunCancellationBetweenIterations() + public async Task Vector54TurnConformanceRunCancellationBetweenIterations() { string vectorJson = """ { @@ -2398,7 +2453,7 @@ public async Task Vector53TurnConformanceRunCancellationBetweenIterations() } [Fact] - public async Task Vector54TurnConformanceRunCancellationBetweenTools() + public async Task Vector55TurnConformanceRunCancellationBetweenTools() { string vectorJson = """ { @@ -2539,7 +2594,7 @@ public async Task Vector54TurnConformanceRunCancellationBetweenTools() } [Fact] - public async Task Vector55TurnConformanceRunContextNoTrimWhenFits() + public async Task Vector56TurnConformanceRunContextNoTrimWhenFits() { string vectorJson = """ { @@ -2612,7 +2667,7 @@ public async Task Vector55TurnConformanceRunContextNoTrimWhenFits() } [Fact] - public async Task Vector56TurnConformanceRunContextPreservesSystemMessages() + public async Task Vector57TurnConformanceRunContextPreservesSystemMessages() { string vectorJson = """ { @@ -2721,7 +2776,7 @@ public async Task Vector56TurnConformanceRunContextPreservesSystemMessages() } [Fact] - public async Task Vector57TurnConformanceRunContextTrimBasic() + public async Task Vector58TurnConformanceRunContextTrimBasic() { string vectorJson = """ { @@ -2881,7 +2936,7 @@ public async Task Vector57TurnConformanceRunContextTrimBasic() } [Fact] - public async Task Vector58TurnConformanceRunEmptyToolResult() + public async Task Vector59TurnConformanceRunEmptyToolResult() { string vectorJson = """ { @@ -2998,7 +3053,7 @@ public async Task Vector58TurnConformanceRunEmptyToolResult() } [Fact] - public async Task Vector59TurnConformanceRunEventsBasicToolLoop() + public async Task Vector60TurnConformanceRunEventsBasicToolLoop() { string vectorJson = """ { @@ -3146,7 +3201,7 @@ public async Task Vector59TurnConformanceRunEventsBasicToolLoop() } [Fact] - public async Task Vector60TurnConformanceRunEventsErrorLogged() + public async Task Vector61TurnConformanceRunEventsErrorLogged() { string vectorJson = """ { @@ -3274,7 +3329,7 @@ public async Task Vector60TurnConformanceRunEventsErrorLogged() } [Fact] - public async Task Vector61TurnConformanceRunEventsNoTools() + public async Task Vector62TurnConformanceRunEventsNoTools() { string vectorJson = """ { @@ -3354,7 +3409,7 @@ public async Task Vector61TurnConformanceRunEventsNoTools() } [Fact] - public async Task Vector62TurnConformanceRunGuardrailAllPass() + public async Task Vector63TurnConformanceRunGuardrailAllPass() { string vectorJson = """ { @@ -3479,7 +3534,7 @@ public async Task Vector62TurnConformanceRunGuardrailAllPass() } [Fact] - public async Task Vector63TurnConformanceRunGuardrailInputDeny() + public async Task Vector64TurnConformanceRunGuardrailInputDeny() { string vectorJson = """ { @@ -3537,7 +3592,7 @@ public async Task Vector63TurnConformanceRunGuardrailInputDeny() } [Fact] - public async Task Vector64TurnConformanceRunGuardrailOutputDeny() + public async Task Vector65TurnConformanceRunGuardrailOutputDeny() { string vectorJson = """ { @@ -3617,7 +3672,7 @@ public async Task Vector64TurnConformanceRunGuardrailOutputDeny() } [Fact] - public async Task Vector65TurnConformanceRunGuardrailToolDeny() + public async Task Vector66TurnConformanceRunGuardrailToolDeny() { string vectorJson = """ { @@ -3773,7 +3828,7 @@ public async Task Vector65TurnConformanceRunGuardrailToolDeny() } [Fact] - public async Task Vector66TurnConformanceRunMaxIterationsExceeded() + public async Task Vector67TurnConformanceRunMaxIterationsExceeded() { string vectorJson = """ { @@ -4285,7 +4340,7 @@ public async Task Vector66TurnConformanceRunMaxIterationsExceeded() } [Fact] - public async Task Vector67TurnConformanceRunMultiTurnToolCalls() + public async Task Vector68TurnConformanceRunMultiTurnToolCalls() { string vectorJson = """ { @@ -4464,7 +4519,7 @@ public async Task Vector67TurnConformanceRunMultiTurnToolCalls() } [Fact] - public async Task Vector68TurnConformanceRunMultipleToolCallsSingleTurn() + public async Task Vector69TurnConformanceRunMultipleToolCallsSingleTurn() { string vectorJson = """ { @@ -4647,7 +4702,7 @@ public async Task Vector68TurnConformanceRunMultipleToolCallsSingleTurn() } [Fact] - public async Task Vector69TurnConformanceRunNoToolCalls() + public async Task Vector70TurnConformanceRunNoToolCalls() { string vectorJson = """ { @@ -4717,7 +4772,7 @@ public async Task Vector69TurnConformanceRunNoToolCalls() } [Fact] - public async Task Vector70TurnConformanceRunParallelToolsBasic() + public async Task Vector71TurnConformanceRunParallelToolsBasic() { string vectorJson = """ { @@ -4893,7 +4948,7 @@ public async Task Vector70TurnConformanceRunParallelToolsBasic() } [Fact] - public async Task Vector71TurnConformanceRunParallelToolsWithGuardrailDeny() + public async Task Vector72TurnConformanceRunParallelToolsWithGuardrailDeny() { string vectorJson = """ { @@ -5087,7 +5142,7 @@ public async Task Vector71TurnConformanceRunParallelToolsWithGuardrailDeny() } [Fact] - public async Task Vector72TurnConformanceRunSingleToolCall() + public async Task Vector73TurnConformanceRunSingleToolCall() { string vectorJson = """ { @@ -5236,7 +5291,7 @@ public async Task Vector72TurnConformanceRunSingleToolCall() } [Fact] - public async Task Vector73TurnConformanceRunSteeringInjectMessage() + public async Task Vector74TurnConformanceRunSteeringInjectMessage() { string vectorJson = """ { @@ -5446,7 +5501,7 @@ public async Task Vector73TurnConformanceRunSteeringInjectMessage() } [Fact] - public async Task Vector74TurnConformanceRunSteeringMultipleMessages() + public async Task Vector75TurnConformanceRunSteeringMultipleMessages() { string vectorJson = """ { @@ -5618,7 +5673,7 @@ public async Task Vector74TurnConformanceRunSteeringMultipleMessages() } [Fact] - public async Task Vector75TurnConformanceRunToolNotRegisteredError() + public async Task Vector76TurnConformanceRunToolNotRegisteredError() { string vectorJson = """ { @@ -5704,7 +5759,7 @@ public async Task Vector75TurnConformanceRunToolNotRegisteredError() } [Fact] - public async Task Vector76TurnConformanceRunToolResultMessageFormat() + public async Task Vector77TurnConformanceRunToolResultMessageFormat() { string vectorJson = """ { @@ -5827,7 +5882,7 @@ public async Task Vector76TurnConformanceRunToolResultMessageFormat() } [Fact] - public async Task Vector77TurnConformanceRunTurnCancelBeforeContext() + public async Task Vector78TurnConformanceRunTurnCancelBeforeContext() { string vectorJson = """ { @@ -5861,7 +5916,7 @@ public async Task Vector77TurnConformanceRunTurnCancelBeforeContext() } [Fact] - public async Task Vector78TurnConformanceRunTurnDelegatedProviderState() + public async Task Vector79TurnConformanceRunTurnDelegatedProviderState() { string vectorJson = """ { @@ -5930,7 +5985,7 @@ public async Task Vector78TurnConformanceRunTurnDelegatedProviderState() } [Fact] - public async Task Vector79TurnConformanceRunTurnFinalOutput() + public async Task Vector80TurnConformanceRunTurnFinalOutput() { string vectorJson = """ { @@ -5977,7 +6032,7 @@ public async Task Vector79TurnConformanceRunTurnFinalOutput() } [Fact] - public async Task Vector80TurnConformanceRunTurnOrderedToolRound() + public async Task Vector81TurnConformanceRunTurnOrderedToolRound() { string vectorJson = """ { @@ -6070,7 +6125,7 @@ public async Task Vector80TurnConformanceRunTurnOrderedToolRound() } [Fact] - public async Task Vector81TurnConformanceRunTurnPermissionDenialIsModelVisible() + public async Task Vector82TurnConformanceRunTurnPermissionDenialIsModelVisible() { string vectorJson = """ { @@ -6123,7 +6178,7 @@ public async Task Vector81TurnConformanceRunTurnPermissionDenialIsModelVisible() } [Fact] - public async Task Vector82WireConformanceToRequestAnthropicImageFormat() + public async Task Vector83WireConformanceToRequestAnthropicImageFormat() { string vectorJson = """ { @@ -6195,7 +6250,7 @@ public async Task Vector82WireConformanceToRequestAnthropicImageFormat() } [Fact] - public async Task Vector83WireConformanceToRequestAnthropicMaxTokensRequired() + public async Task Vector84WireConformanceToRequestAnthropicMaxTokensRequired() { string vectorJson = """ { @@ -6252,7 +6307,7 @@ public async Task Vector83WireConformanceToRequestAnthropicMaxTokensRequired() } [Fact] - public async Task Vector84WireConformanceToRequestAnthropicOptions() + public async Task Vector85WireConformanceToRequestAnthropicOptions() { string vectorJson = """ { @@ -6323,7 +6378,7 @@ public async Task Vector84WireConformanceToRequestAnthropicOptions() } [Fact] - public async Task Vector85WireConformanceToRequestAnthropicSystemSeparate() + public async Task Vector86WireConformanceToRequestAnthropicSystemSeparate() { string vectorJson = """ { @@ -6392,7 +6447,7 @@ public async Task Vector85WireConformanceToRequestAnthropicSystemSeparate() } [Fact] - public async Task Vector86WireConformanceToRequestAnthropicToolWire() + public async Task Vector87WireConformanceToRequestAnthropicToolWire() { string vectorJson = """ { @@ -6479,7 +6534,7 @@ public async Task Vector86WireConformanceToRequestAnthropicToolWire() } [Fact] - public async Task Vector87WireConformanceToRequestChatAudioMp3() + public async Task Vector88WireConformanceToRequestChatAudioMp3() { string vectorJson = """ { @@ -6539,7 +6594,7 @@ public async Task Vector87WireConformanceToRequestChatAudioMp3() } [Fact] - public async Task Vector88WireConformanceToRequestChatAudioPart() + public async Task Vector89WireConformanceToRequestChatAudioPart() { string vectorJson = """ { @@ -6599,7 +6654,7 @@ public async Task Vector88WireConformanceToRequestChatAudioPart() } [Fact] - public async Task Vector89WireConformanceToRequestChatImageBase64() + public async Task Vector90WireConformanceToRequestChatImageBase64() { string vectorJson = """ { @@ -6665,7 +6720,7 @@ public async Task Vector89WireConformanceToRequestChatImageBase64() } [Fact] - public async Task Vector90WireConformanceToRequestChatImagePart() + public async Task Vector91WireConformanceToRequestChatImagePart() { string vectorJson = """ { @@ -6731,7 +6786,7 @@ public async Task Vector90WireConformanceToRequestChatImagePart() } [Fact] - public async Task Vector91WireConformanceToRequestChatMultipartContent() + public async Task Vector92WireConformanceToRequestChatMultipartContent() { string vectorJson = """ { @@ -6797,7 +6852,7 @@ public async Task Vector91WireConformanceToRequestChatMultipartContent() } [Fact] - public async Task Vector92WireConformanceToRequestChatSimple() + public async Task Vector93WireConformanceToRequestChatSimple() { string vectorJson = """ { @@ -6861,7 +6916,7 @@ public async Task Vector92WireConformanceToRequestChatSimple() } [Fact] - public async Task Vector93WireConformanceToRequestChatSingleTextOptimized() + public async Task Vector94WireConformanceToRequestChatSingleTextOptimized() { string vectorJson = """ { @@ -6912,7 +6967,7 @@ public async Task Vector93WireConformanceToRequestChatSingleTextOptimized() } [Fact] - public async Task Vector94WireConformanceToRequestChatWithOptions() + public async Task Vector95WireConformanceToRequestChatWithOptions() { string vectorJson = """ { @@ -6982,7 +7037,7 @@ public async Task Vector94WireConformanceToRequestChatWithOptions() } [Fact] - public async Task Vector95WireConformanceToRequestEmbeddingWire() + public async Task Vector96WireConformanceToRequestEmbeddingWire() { string vectorJson = """ { @@ -7028,7 +7083,7 @@ public async Task Vector95WireConformanceToRequestEmbeddingWire() } [Fact] - public async Task Vector96WireConformanceToRequestImageWire() + public async Task Vector97WireConformanceToRequestImageWire() { string vectorJson = """ { @@ -7074,7 +7129,7 @@ public async Task Vector96WireConformanceToRequestImageWire() } [Fact] - public async Task Vector97WireConformanceToRequestKindToJsonTypeMapping() + public async Task Vector98WireConformanceToRequestKindToJsonTypeMapping() { string vectorJson = """ { @@ -7195,7 +7250,7 @@ public async Task Vector97WireConformanceToRequestKindToJsonTypeMapping() } [Fact] - public async Task Vector98WireConformanceToRequestOptionsAdditionalProperties() + public async Task Vector99WireConformanceToRequestOptionsAdditionalProperties() { string vectorJson = """ { @@ -7251,7 +7306,7 @@ public async Task Vector98WireConformanceToRequestOptionsAdditionalProperties() } [Fact] - public async Task Vector99WireConformanceToRequestOptionsMaxCompletionTokens() + public async Task Vector100WireConformanceToRequestOptionsMaxCompletionTokens() { string vectorJson = """ { @@ -7305,7 +7360,7 @@ public async Task Vector99WireConformanceToRequestOptionsMaxCompletionTokens() } [Fact] - public async Task Vector100WireConformanceToRequestOptionsStopSequences() + public async Task Vector101WireConformanceToRequestOptionsStopSequences() { string vectorJson = """ { @@ -7365,7 +7420,7 @@ public async Task Vector100WireConformanceToRequestOptionsStopSequences() } [Fact] - public async Task Vector101WireConformanceToRequestResponsesSimple() + public async Task Vector102WireConformanceToRequestResponsesSimple() { string vectorJson = """ { @@ -7426,7 +7481,7 @@ public async Task Vector101WireConformanceToRequestResponsesSimple() } [Fact] - public async Task Vector102WireConformanceToRequestResponsesStructuredOutput() + public async Task Vector103WireConformanceToRequestResponsesStructuredOutput() { string vectorJson = """ { @@ -7515,7 +7570,7 @@ public async Task Vector102WireConformanceToRequestResponsesStructuredOutput() } [Fact] - public async Task Vector103WireConformanceToRequestResponsesWithTools() + public async Task Vector104WireConformanceToRequestResponsesWithTools() { string vectorJson = """ { @@ -7597,7 +7652,7 @@ public async Task Vector103WireConformanceToRequestResponsesWithTools() } [Fact] - public async Task Vector104WireConformanceToRequestStructuredOutput() + public async Task Vector105WireConformanceToRequestStructuredOutput() { string vectorJson = """ { @@ -7682,7 +7737,7 @@ public async Task Vector104WireConformanceToRequestStructuredOutput() } [Fact] - public async Task Vector105WireConformanceToRequestStructuredOutputNestedOptional() + public async Task Vector106WireConformanceToRequestStructuredOutputNestedOptional() { string vectorJson = """ { @@ -7786,7 +7841,7 @@ public async Task Vector105WireConformanceToRequestStructuredOutputNestedOptiona } [Fact] - public async Task Vector106WireConformanceToRequestToolsBindingsStripped() + public async Task Vector107WireConformanceToRequestToolsBindingsStripped() { string vectorJson = """ { @@ -7880,7 +7935,7 @@ public async Task Vector106WireConformanceToRequestToolsBindingsStripped() } [Fact] - public async Task Vector107WireConformanceToRequestToolsFunctionWire() + public async Task Vector108WireConformanceToRequestToolsFunctionWire() { string vectorJson = """ { @@ -7964,7 +8019,7 @@ public async Task Vector107WireConformanceToRequestToolsFunctionWire() } [Fact] - public async Task Vector108WireConformanceToRequestToolsNullWhenEmpty() + public async Task Vector109WireConformanceToRequestToolsNullWhenEmpty() { string vectorJson = """ { @@ -8015,7 +8070,7 @@ public async Task Vector108WireConformanceToRequestToolsNullWhenEmpty() } [Fact] - public async Task Vector109WireConformanceToRequestToolsStrictMode() + public async Task Vector110WireConformanceToRequestToolsStrictMode() { string vectorJson = """ { diff --git a/runtime/csharp/Prompty.Core/Model/LiveChatConformance.cs b/runtime/csharp/Prompty.Core/Model/LiveChatConformance.cs new file mode 100644 index 000000000..a6f614380 --- /dev/null +++ b/runtime/csharp/Prompty.Core/Model/LiveChatConformance.cs @@ -0,0 +1,31 @@ +// +// Copyright (c) Microsoft. All rights reserved. +#nullable enable + +using System.Threading; +using System.Threading.Tasks; + +#pragma warning disable IDE0130 +namespace Prompty.Core; +#pragma warning restore IDE0130 + + /// + /// Conformance-only seam hosting the "service drill" vectors: a conformance + /// + /// vector that crosses the REAL Executor->Processor transport seam instead of + /// + /// stopping at the SDK edge. A drill runs against a live provider when + /// + /// credentials are present and REPLAYS a recorded cassette when they are not; + /// + /// the live<->replay parity is the gate. + /// + /// Like every other conformance seam this interface exists purely to carry + /// + public interface ILiveChatConformance + { + /// + /// Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes + /// + Task CompleteAsync(object input); + } diff --git a/runtime/go/prompty/model/live_chat_conformance.go b/runtime/go/prompty/model/live_chat_conformance.go new file mode 100644 index 000000000..b545fe189 --- /dev/null +++ b/runtime/go/prompty/model/live_chat_conformance.go @@ -0,0 +1,17 @@ +// +// Code generated by Typra emitter; DO NOT EDIT. + +package prompty + +// LiveChatConformance represents Conformance-only seam hosting the "service drill" vectors: a conformance +// vector that crosses the REAL Executor->Processor transport seam instead of +// stopping at the SDK edge. A drill runs against a live provider when +// credentials are present and REPLAYS a recorded cassette when they are not; +// the live<->replay parity is the gate. +// +// Like every other conformance seam this interface exists purely to carry + +type LiveChatConformance interface { + // Complete — Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes + Complete(input interface{}) (interface{}, error) +} diff --git a/runtime/go/prompty/model/vector_conformance_test.go b/runtime/go/prompty/model/vector_conformance_test.go index 0c8687e17..09d5a5b55 100644 --- a/runtime/go/prompty/model/vector_conformance_test.go +++ b/runtime/go/prompty/model/vector_conformance_test.go @@ -560,7 +560,61 @@ func TestVector15DiscoveryConformanceMapModelOpenaiModelFinetuneNoOwner(t *testi vectorrunner.RunVector(t, "DiscoveryConformance", "mapModel", vector, vcSeam()) } -func TestVector16LoadConformanceLoadBasicLoad(t *testing.T) { +func TestVector16LiveChatConformanceCompleteOpenaiChatDrill(t *testing.T) { + vectorJSON := `{ + "name": "openai_chat_drill", + "stage": "drill", + "provider": "openai", + "input": { + "provider": "openai", + "model": "gpt-4o-mini", + "apiKey": { + "$env": "OPENAI_API_KEY" + }, + "endpoint": { + "$env": "OPENAI_BASE_URL" + }, + "messages": [ + { + "role": "user", + "content": "Say hello in exactly one word." + } + ], + "options": { + "temperature": 0, + "maxOutputTokens": 16 + } + }, + "exchange": { + "transport": { + "baseUrl": { + "$env": "OPENAI_BASE_URL" + } + }, + "cassette": { + "$json": "cassettes/openai_chat_drill.json" + }, + "planes": [ + "transport", + "wire", + "semantic" + ] + }, + "expected": { + "role": "assistant", + "contentNonEmpty": true, + "finishReasonInEnum": true + }, + "operation": "complete" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LiveChatConformance", "complete", vector, vcSeam()) +} + +func TestVector17LoadConformanceLoadBasicLoad(t *testing.T) { vectorJSON := `{ "name": "basic_load", "description": "Load basic.prompty with env var resolution and verify all PromptAgent fields", @@ -630,7 +684,7 @@ func TestVector16LoadConformanceLoadBasicLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector17LoadConformanceLoadConnectionTypesLoad(t *testing.T) { +func TestVector18LoadConformanceLoadConnectionTypesLoad(t *testing.T) { vectorJSON := `{ "name": "connection_types_load", "description": "All connection kinds (key, reference, remote, anonymous) load correctly", @@ -665,7 +719,7 @@ func TestVector17LoadConformanceLoadConnectionTypesLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector18LoadConformanceLoadEmbeddingLoad(t *testing.T) { +func TestVector19LoadConformanceLoadEmbeddingLoad(t *testing.T) { vectorJSON := `{ "name": "embedding_load", "description": "Load embedding.prompty and verify apiType=embedding with correct model id", @@ -707,7 +761,7 @@ func TestVector18LoadConformanceLoadEmbeddingLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector19LoadConformanceLoadEmptyFrontmatterBodyOnly(t *testing.T) { +func TestVector20LoadConformanceLoadEmptyFrontmatterBodyOnly(t *testing.T) { vectorJSON := `{ "name": "empty_frontmatter_body_only", "description": "Empty frontmatter with body — instructions extracted, defaults applied", @@ -733,7 +787,7 @@ func TestVector19LoadConformanceLoadEmptyFrontmatterBodyOnly(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector20LoadConformanceLoadEnvDefault(t *testing.T) { +func TestVector21LoadConformanceLoadEnvDefault(t *testing.T) { vectorJSON := `{ "name": "env_default", "description": "${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set", @@ -768,7 +822,7 @@ func TestVector20LoadConformanceLoadEnvDefault(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector21LoadConformanceLoadEnvMissingError(t *testing.T) { +func TestVector22LoadConformanceLoadEnvMissingError(t *testing.T) { vectorJSON := `{ "name": "env_missing_error", "description": "${env:NONEXISTENT} with no default and var not set raises an error", @@ -798,7 +852,7 @@ func TestVector21LoadConformanceLoadEnvMissingError(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector22LoadConformanceLoadEnvResolution(t *testing.T) { +func TestVector23LoadConformanceLoadEnvResolution(t *testing.T) { vectorJSON := `{ "name": "env_resolution", "description": "${env:MY_VAR} resolves to the environment variable value when set", @@ -835,7 +889,7 @@ func TestVector22LoadConformanceLoadEnvResolution(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector23LoadConformanceLoadFileReferenceInTreeAllowed(t *testing.T) { +func TestVector24LoadConformanceLoadFileReferenceInTreeAllowed(t *testing.T) { vectorJSON := `{ "name": "file_reference_in_tree_allowed", "description": "A ${file:sub/data.json} reference that stays within the prompt file's directory tree resolves to the parsed content (containment control alongside the traversal-reject vector)", @@ -874,7 +928,7 @@ func TestVector23LoadConformanceLoadFileReferenceInTreeAllowed(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector24LoadConformanceLoadFileReferenceTraversalRejected(t *testing.T) { +func TestVector25LoadConformanceLoadFileReferenceTraversalRejected(t *testing.T) { vectorJSON := "{\n \"name\": \"file_reference_traversal_rejected\",\n \"description\": \"A ${file:../secret.json} reference that escapes the prompt file's directory via `..` is rejected even though the target file exists, preventing arbitrary file disclosure (GHSA-7pfc-5v9r-j299). Uses forward-slash separators so the vector holds on every OS.\",\n \"stage\": \"load\",\n \"input\": {\n \"agent_subdir\": \"app\",\n \"frontmatter\": {\n \"name\": \"file-traversal-test\",\n \"model\": {\n \"id\": \"gpt-4\",\n \"connection\": \"${file:../secret.json}\"\n }\n },\n \"files\": {\n \"../secret.json\": {\n \"kind\": \"key\",\n \"endpoint\": \"https://attacker.example.com\",\n \"apiKey\": \"leaked\"\n }\n }\n },\n \"expectedError\": {\n \"kind\": \"file_reference\"\n },\n \"operation\": \"load\"\n}" var vector map[string]any if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { @@ -883,7 +937,7 @@ func TestVector24LoadConformanceLoadFileReferenceTraversalRejected(t *testing.T) vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector25LoadConformanceLoadFileResolution(t *testing.T) { +func TestVector26LoadConformanceLoadFileResolution(t *testing.T) { vectorJSON := `{ "name": "file_resolution", "description": "${file:shared_connection.json} resolves to the parsed JSON content of the referenced file", @@ -924,7 +978,7 @@ func TestVector25LoadConformanceLoadFileResolution(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector26LoadConformanceLoadImageApitypeLoad(t *testing.T) { +func TestVector27LoadConformanceLoadImageApitypeLoad(t *testing.T) { vectorJSON := `{ "name": "image_apitype_load", "description": "An image-generation prompt loads with apiType image and passthrough additionalProperties model options", @@ -969,7 +1023,7 @@ func TestVector26LoadConformanceLoadImageApitypeLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector27LoadConformanceLoadInputScalarShorthand(t *testing.T) { +func TestVector28LoadConformanceLoadInputScalarShorthand(t *testing.T) { vectorJSON := `{ "name": "input_scalar_shorthand", "description": "Scalar input values create typed Property with default set", @@ -1015,7 +1069,7 @@ func TestVector27LoadConformanceLoadInputScalarShorthand(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector28LoadConformanceLoadInputValidationDefaultFill(t *testing.T) { +func TestVector29LoadConformanceLoadInputValidationDefaultFill(t *testing.T) { vectorJSON := `{ "name": "input_validation_default_fill", "description": "An input with a default value is filled in when no value is provided", @@ -1048,7 +1102,7 @@ func TestVector28LoadConformanceLoadInputValidationDefaultFill(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector29LoadConformanceLoadInputValidationExampleNotUsed(t *testing.T) { +func TestVector30LoadConformanceLoadInputValidationExampleNotUsed(t *testing.T) { vectorJSON := `{ "name": "input_validation_example_not_used", "description": "An input with example='sample' but no default must NOT use the example at runtime", @@ -1080,7 +1134,7 @@ func TestVector29LoadConformanceLoadInputValidationExampleNotUsed(t *testing.T) vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector30LoadConformanceLoadInputValidationOptionalOmit(t *testing.T) { +func TestVector31LoadConformanceLoadInputValidationOptionalOmit(t *testing.T) { vectorJSON := `{ "name": "input_validation_optional_omit", "description": "A non-required input with no default and no value is omitted (not an error)", @@ -1111,7 +1165,7 @@ func TestVector30LoadConformanceLoadInputValidationOptionalOmit(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector31LoadConformanceLoadInputValidationRequired(t *testing.T) { +func TestVector32LoadConformanceLoadInputValidationRequired(t *testing.T) { vectorJSON := `{ "name": "input_validation_required", "description": "A required input with no default and no value provided raises ValueError", @@ -1143,7 +1197,7 @@ func TestVector31LoadConformanceLoadInputValidationRequired(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector32LoadConformanceLoadInstructionsFromBody(t *testing.T) { +func TestVector33LoadConformanceLoadInstructionsFromBody(t *testing.T) { vectorJSON := `{ "name": "instructions_from_body", "description": "The markdown body after the closing --- becomes the instructions field", @@ -1167,7 +1221,7 @@ func TestVector32LoadConformanceLoadInstructionsFromBody(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector33LoadConformanceLoadInvalidFrontmatterError(t *testing.T) { +func TestVector34LoadConformanceLoadInvalidFrontmatterError(t *testing.T) { vectorJSON := `{ "name": "invalid_frontmatter_error", "description": "Malformed YAML in frontmatter raises a parse error", @@ -1187,7 +1241,7 @@ func TestVector33LoadConformanceLoadInvalidFrontmatterError(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector34LoadConformanceLoadKindAlwaysPrompt(t *testing.T) { +func TestVector35LoadConformanceLoadKindAlwaysPrompt(t *testing.T) { vectorJSON := `{ "name": "kind_always_prompt", "description": "The loader always injects kind='prompt' — .prompty files never specify kind themselves", @@ -1207,7 +1261,7 @@ func TestVector34LoadConformanceLoadKindAlwaysPrompt(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector35LoadConformanceLoadMinimalLoad(t *testing.T) { +func TestVector36LoadConformanceLoadMinimalLoad(t *testing.T) { vectorJSON := `{ "name": "minimal_load", "description": "Load minimal.prompty with model shorthand and bare-minimum frontmatter", @@ -1235,7 +1289,7 @@ func TestVector35LoadConformanceLoadMinimalLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector36LoadConformanceLoadMissingFileError(t *testing.T) { +func TestVector37LoadConformanceLoadMissingFileError(t *testing.T) { vectorJSON := `{ "name": "missing_file_error", "description": "Loading a nonexistent .prompty file raises FileNotFoundError", @@ -1255,7 +1309,7 @@ func TestVector36LoadConformanceLoadMissingFileError(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector37LoadConformanceLoadModelShorthand(t *testing.T) { +func TestVector38LoadConformanceLoadModelShorthand(t *testing.T) { vectorJSON := `{ "name": "model_shorthand", "description": "String shorthand 'model: gpt-4o' expands to a Model object with id set", @@ -1282,7 +1336,7 @@ func TestVector37LoadConformanceLoadModelShorthand(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector38LoadConformanceLoadStructuredOutputsLoad(t *testing.T) { +func TestVector39LoadConformanceLoadStructuredOutputsLoad(t *testing.T) { vectorJSON := `{ "name": "structured_outputs_load", "description": "Outputs load as a list of properties preserving kind, description, and required flags", @@ -1349,7 +1403,7 @@ func TestVector38LoadConformanceLoadStructuredOutputsLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector39LoadConformanceLoadTemplateStringInvalid(t *testing.T) { +func TestVector40LoadConformanceLoadTemplateStringInvalid(t *testing.T) { vectorJSON := `{ "name": "template_string_invalid", "description": "Template as a bare string is not valid v2 — must be an object with format/parser", @@ -1375,7 +1429,7 @@ func TestVector39LoadConformanceLoadTemplateStringInvalid(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector40LoadConformanceLoadToolsCustomLoad(t *testing.T) { +func TestVector41LoadConformanceLoadToolsCustomLoad(t *testing.T) { vectorJSON := `{ "name": "tools_custom_load", "description": "Load a prompty with an unknown tool kind — falls through to CustomTool", @@ -1413,7 +1467,7 @@ func TestVector40LoadConformanceLoadToolsCustomLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector41LoadConformanceLoadToolsFunctionLoad(t *testing.T) { +func TestVector42LoadConformanceLoadToolsFunctionLoad(t *testing.T) { vectorJSON := `{ "name": "tools_function_load", "description": "Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings", @@ -1480,7 +1534,7 @@ func TestVector41LoadConformanceLoadToolsFunctionLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector42LoadConformanceLoadToolsMcpLoad(t *testing.T) { +func TestVector43LoadConformanceLoadToolsMcpLoad(t *testing.T) { vectorJSON := `{ "name": "tools_mcp_load", "description": "Load a prompty with an MCP tool — kind, serverName, connection preserved", @@ -1520,7 +1574,7 @@ func TestVector42LoadConformanceLoadToolsMcpLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector43LoadConformanceLoadToolsOpenapiLoad(t *testing.T) { +func TestVector44LoadConformanceLoadToolsOpenapiLoad(t *testing.T) { vectorJSON := `{ "name": "tools_openapi_load", "description": "Load a prompty with an OpenAPI tool — specification path preserved", @@ -1560,7 +1614,7 @@ func TestVector43LoadConformanceLoadToolsOpenapiLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector44TurnConformanceReplayMaxIterations(t *testing.T) { +func TestVector45TurnConformanceReplayMaxIterations(t *testing.T) { vectorJSON := `{ "name": "max_iterations", "stage": "replay", @@ -1596,7 +1650,7 @@ func TestVector44TurnConformanceReplayMaxIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector45TurnConformanceReplayNoTool(t *testing.T) { +func TestVector46TurnConformanceReplayNoTool(t *testing.T) { vectorJSON := `{ "name": "no_tool", "stage": "replay", @@ -1628,7 +1682,7 @@ func TestVector45TurnConformanceReplayNoTool(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector46TurnConformanceReplayPermissionDenied(t *testing.T) { +func TestVector47TurnConformanceReplayPermissionDenied(t *testing.T) { vectorJSON := `{ "name": "permission_denied", "stage": "replay", @@ -1663,7 +1717,7 @@ func TestVector46TurnConformanceReplayPermissionDenied(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector47TurnConformanceReplayToolFailure(t *testing.T) { +func TestVector48TurnConformanceReplayToolFailure(t *testing.T) { vectorJSON := `{ "name": "tool_failure", "stage": "replay", @@ -1700,7 +1754,7 @@ func TestVector47TurnConformanceReplayToolFailure(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector48TurnConformanceReplayToolSuccess(t *testing.T) { +func TestVector49TurnConformanceReplayToolSuccess(t *testing.T) { vectorJSON := `{ "name": "tool_success", "stage": "replay", @@ -1737,7 +1791,7 @@ func TestVector48TurnConformanceReplayToolSuccess(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector49TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { +func TestVector50TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorJSON := `{ "name": "assistant_tool_calls_metadata", "description": "When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.", @@ -1890,7 +1944,7 @@ func TestVector49TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector50TurnConformanceRunAsyncToolFunction(t *testing.T) { +func TestVector51TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorJSON := `{ "name": "async_tool_function", "description": "Agent loop correctly handles async tool functions", @@ -2001,7 +2055,7 @@ func TestVector50TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector51TurnConformanceRunBindingsInjected(t *testing.T) { +func TestVector52TurnConformanceRunBindingsInjected(t *testing.T) { vectorJSON := `{ "name": "bindings_injected", "description": "Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.", @@ -2132,7 +2186,7 @@ func TestVector51TurnConformanceRunBindingsInjected(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector52TurnConformanceRunCancellationBeforeLlm(t *testing.T) { +func TestVector53TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorJSON := `{ "name": "cancellation_before_llm", "description": "§13.2 Cancellation — Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.", @@ -2194,7 +2248,7 @@ func TestVector52TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector53TurnConformanceRunCancellationBetweenIterations(t *testing.T) { +func TestVector54TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_iterations", "description": "§13.2 Cancellation — Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.", @@ -2325,7 +2379,7 @@ func TestVector53TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector54TurnConformanceRunCancellationBetweenTools(t *testing.T) { +func TestVector55TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_tools", "description": "§13.2 Cancellation — LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.", @@ -2465,7 +2519,7 @@ func TestVector54TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector55TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { +func TestVector56TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorJSON := `{ "name": "context_no_trim_when_fits", "description": "§13.3 Context Window — Messages fit within the context budget. No trimming occurs and all messages are preserved.", @@ -2537,7 +2591,7 @@ func TestVector55TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector56TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { +func TestVector57TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { vectorJSON := `{ "name": "context_preserves_system_messages", "description": "§13.3 Context Window — Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.", @@ -2645,7 +2699,7 @@ func TestVector56TurnConformanceRunContextPreservesSystemMessages(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector57TurnConformanceRunContextTrimBasic(t *testing.T) { +func TestVector58TurnConformanceRunContextTrimBasic(t *testing.T) { vectorJSON := `{ "name": "context_trim_basic", "description": "§13.3 Context Window — Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.", @@ -2804,7 +2858,7 @@ func TestVector57TurnConformanceRunContextTrimBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector58TurnConformanceRunEmptyToolResult(t *testing.T) { +func TestVector59TurnConformanceRunEmptyToolResult(t *testing.T) { vectorJSON := `{ "name": "empty_tool_result", "description": "Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content — it must not be skipped or filtered out.", @@ -2920,7 +2974,7 @@ func TestVector58TurnConformanceRunEmptyToolResult(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector59TurnConformanceRunEventsBasicToolLoop(t *testing.T) { +func TestVector60TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorJSON := `{ "name": "events_basic_tool_loop", "description": "§13.1 Events — A 2-turn tool-call loop emits the full event lifecycle: status → tool_call_start → tool_result → messages_updated → done.", @@ -3067,7 +3121,7 @@ func TestVector59TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector60TurnConformanceRunEventsErrorLogged(t *testing.T) { +func TestVector61TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorJSON := `{ "name": "events_error_logged", "description": "§13.1 Events — A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.", @@ -3194,7 +3248,7 @@ func TestVector60TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector61TurnConformanceRunEventsNoTools(t *testing.T) { +func TestVector62TurnConformanceRunEventsNoTools(t *testing.T) { vectorJSON := `{ "name": "events_no_tools", "description": "§13.1 Events — Single-turn completion with no tool calls. Only the done event is emitted.", @@ -3273,7 +3327,7 @@ func TestVector61TurnConformanceRunEventsNoTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector62TurnConformanceRunGuardrailAllPass(t *testing.T) { +func TestVector63TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorJSON := `{ "name": "guardrail_all_pass", "description": "§13.4 Guardrails — All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.", @@ -3397,7 +3451,7 @@ func TestVector62TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector63TurnConformanceRunGuardrailInputDeny(t *testing.T) { +func TestVector64TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_input_deny", "description": "§13.4 Guardrails — Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.", @@ -3454,7 +3508,7 @@ func TestVector63TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector64TurnConformanceRunGuardrailOutputDeny(t *testing.T) { +func TestVector65TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_output_deny", "description": "§13.4 Guardrails — Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.", @@ -3533,7 +3587,7 @@ func TestVector64TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector65TurnConformanceRunGuardrailToolDeny(t *testing.T) { +func TestVector66TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_tool_deny", "description": "§13.4 Guardrails — LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.", @@ -3688,7 +3742,7 @@ func TestVector65TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector66TurnConformanceRunMaxIterationsExceeded(t *testing.T) { +func TestVector67TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorJSON := `{ "name": "max_iterations_exceeded", "description": "LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.", @@ -4199,7 +4253,7 @@ func TestVector66TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector67TurnConformanceRunMultiTurnToolCalls(t *testing.T) { +func TestVector68TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorJSON := `{ "name": "multi_turn_tool_calls", "description": "Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.", @@ -4377,7 +4431,7 @@ func TestVector67TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector68TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { +func TestVector69TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorJSON := `{ "name": "multiple_tool_calls_single_turn", "description": "Turn 1: LLM returns two parallel tool calls — get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.", @@ -4559,7 +4613,7 @@ func TestVector68TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector69TurnConformanceRunNoToolCalls(t *testing.T) { +func TestVector70TurnConformanceRunNoToolCalls(t *testing.T) { vectorJSON := `{ "name": "no_tool_calls", "description": "LLM returns content immediately with no tool calls — the agent loop completes in a single iteration.", @@ -4628,7 +4682,7 @@ func TestVector69TurnConformanceRunNoToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector70TurnConformanceRunParallelToolsBasic(t *testing.T) { +func TestVector71TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_basic", "description": "§13.6 Parallel Tools — LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.", @@ -4803,7 +4857,7 @@ func TestVector70TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector71TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { +func TestVector72TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_guardrail_deny", "description": "§13.6 Parallel Tools + §13.4 Guardrails — 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.", @@ -4996,7 +5050,7 @@ func TestVector71TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector72TurnConformanceRunSingleToolCall(t *testing.T) { +func TestVector73TurnConformanceRunSingleToolCall(t *testing.T) { vectorJSON := `{ "name": "single_tool_call", "description": "Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.", @@ -5144,7 +5198,7 @@ func TestVector72TurnConformanceRunSingleToolCall(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector73TurnConformanceRunSteeringInjectMessage(t *testing.T) { +func TestVector74TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorJSON := `{ "name": "steering_inject_message", "description": "§13.5 Steering — After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.", @@ -5353,7 +5407,7 @@ func TestVector73TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector74TurnConformanceRunSteeringMultipleMessages(t *testing.T) { +func TestVector75TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorJSON := `{ "name": "steering_multiple_messages", "description": "§13.5 Steering — Two steering messages are queued and both injected before iteration 2. Tests atomic drain — both appear in order.", @@ -5524,7 +5578,7 @@ func TestVector74TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector75TurnConformanceRunToolNotRegisteredError(t *testing.T) { +func TestVector76TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorJSON := `{ "name": "tool_not_registered_error", "description": "LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.", @@ -5609,7 +5663,7 @@ func TestVector75TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector76TurnConformanceRunToolResultMessageFormat(t *testing.T) { +func TestVector77TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorJSON := `{ "name": "tool_result_message_format", "description": "After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.", @@ -5731,7 +5785,7 @@ func TestVector76TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector77TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { +func TestVector78TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorJSON := `{ "name": "cancel_before_context", "stage": "turn", @@ -5764,7 +5818,7 @@ func TestVector77TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector78TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { +func TestVector79TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorJSON := `{ "name": "delegated_provider_state", "stage": "turn", @@ -5832,7 +5886,7 @@ func TestVector78TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector79TurnConformanceRunTurnFinalOutput(t *testing.T) { +func TestVector80TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorJSON := `{ "name": "final_output", "stage": "turn", @@ -5878,7 +5932,7 @@ func TestVector79TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector80TurnConformanceRunTurnOrderedToolRound(t *testing.T) { +func TestVector81TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorJSON := `{ "name": "ordered_tool_round", "stage": "turn", @@ -5970,7 +6024,7 @@ func TestVector80TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector81TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { +func TestVector82TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "permission_denial_is_model_visible", "stage": "turn", @@ -6022,7 +6076,7 @@ func TestVector81TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector82WireConformanceToRequestAnthropicImageFormat(t *testing.T) { +func TestVector83WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorJSON := `{ "name": "anthropic_image_format", "description": "§7.5 — Anthropic: ImagePart uses base64 source block with media_type, not image_url format.", @@ -6093,7 +6147,7 @@ func TestVector82WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector83WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { +func TestVector84WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { vectorJSON := `{ "name": "anthropic_max_tokens_required", "description": "§7.5 — Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).", @@ -6149,7 +6203,7 @@ func TestVector83WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector84WireConformanceToRequestAnthropicOptions(t *testing.T) { +func TestVector85WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorJSON := `{ "name": "anthropic_options", "description": "Anthropic: option names map correctly (topK, stopSequences)", @@ -6219,7 +6273,7 @@ func TestVector84WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector85WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { +func TestVector86WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { vectorJSON := `{ "name": "anthropic_system_separate", "description": "§7.5 — Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.", @@ -6287,7 +6341,7 @@ func TestVector85WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector86WireConformanceToRequestAnthropicToolWire(t *testing.T) { +func TestVector87WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorJSON := `{ "name": "anthropic_tool_wire", "description": "Anthropic: tools use input_schema (not parameters), name at top level", @@ -6373,7 +6427,7 @@ func TestVector86WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector87WireConformanceToRequestChatAudioMp3(t *testing.T) { +func TestVector88WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorJSON := `{ "name": "chat_audio_mp3", "description": "§7.1.2 — AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.", @@ -6432,7 +6486,7 @@ func TestVector87WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector88WireConformanceToRequestChatAudioPart(t *testing.T) { +func TestVector89WireConformanceToRequestChatAudioPart(t *testing.T) { vectorJSON := `{ "name": "chat_audio_part", "description": "§7.1.2 — AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.", @@ -6491,7 +6545,7 @@ func TestVector88WireConformanceToRequestChatAudioPart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector89WireConformanceToRequestChatImageBase64(t *testing.T) { +func TestVector90WireConformanceToRequestChatImageBase64(t *testing.T) { vectorJSON := `{ "name": "chat_image_base64", "description": "Base64 data URI image part preserved as-is in image_url", @@ -6556,7 +6610,7 @@ func TestVector89WireConformanceToRequestChatImageBase64(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector90WireConformanceToRequestChatImagePart(t *testing.T) { +func TestVector91WireConformanceToRequestChatImagePart(t *testing.T) { vectorJSON := `{ "name": "chat_image_part", "description": "Image content part converted to OpenAI image_url format", @@ -6621,7 +6675,7 @@ func TestVector90WireConformanceToRequestChatImagePart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector91WireConformanceToRequestChatMultipartContent(t *testing.T) { +func TestVector92WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorJSON := `{ "name": "chat_multipart_content", "description": "§7.1.1/§7.1.2 — Message with TextPart + ImagePart MUST produce an array of typed content blocks.", @@ -6686,7 +6740,7 @@ func TestVector91WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector92WireConformanceToRequestChatSimple(t *testing.T) { +func TestVector93WireConformanceToRequestChatSimple(t *testing.T) { vectorJSON := `{ "name": "chat_simple", "description": "§7.1.7 — Two messages (system + user), no tools, no options. Minimal chat request.", @@ -6749,7 +6803,7 @@ func TestVector92WireConformanceToRequestChatSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector93WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { +func TestVector94WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { vectorJSON := `{ "name": "chat_single_text_optimized", "description": "§7.1.1 — Message with exactly 1 TextPart MUST produce a plain string content, not an array.", @@ -6799,7 +6853,7 @@ func TestVector93WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector94WireConformanceToRequestChatWithOptions(t *testing.T) { +func TestVector95WireConformanceToRequestChatWithOptions(t *testing.T) { vectorJSON := `{ "name": "chat_with_options", "description": "§7.1.5 — All standard ModelOptions mapped to OpenAI parameter names.", @@ -6868,7 +6922,7 @@ func TestVector94WireConformanceToRequestChatWithOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector95WireConformanceToRequestEmbeddingWire(t *testing.T) { +func TestVector96WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorJSON := `{ "name": "embedding_wire", "description": "§7.2 — Embedding request: text extracted from messages, single string input (not array).", @@ -6913,7 +6967,7 @@ func TestVector95WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector96WireConformanceToRequestImageWire(t *testing.T) { +func TestVector97WireConformanceToRequestImageWire(t *testing.T) { vectorJSON := `{ "name": "image_wire", "description": "§7.3 — Image generation request: prompt extracted from last user message.", @@ -6958,7 +7012,7 @@ func TestVector96WireConformanceToRequestImageWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector97WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { +func TestVector98WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorJSON := `{ "name": "kind_to_json_type_mapping", "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.", @@ -7078,7 +7132,7 @@ func TestVector97WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector98WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { +func TestVector99WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { vectorJSON := `{ "name": "options_additional_properties", "description": "§7.1.5 — additionalProperties from ModelOptions MUST be merged into the request as top-level keys.", @@ -7133,7 +7187,7 @@ func TestVector98WireConformanceToRequestOptionsAdditionalProperties(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector99WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { +func TestVector100WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { vectorJSON := `{ "name": "options_max_completion_tokens", "description": "§7.1.5 — maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.", @@ -7186,7 +7240,7 @@ func TestVector99WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector100WireConformanceToRequestOptionsStopSequences(t *testing.T) { +func TestVector101WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorJSON := `{ "name": "options_stop_sequences", "description": "§7.1.5 — stopSequences MUST map to 'stop' in the OpenAI wire format.", @@ -7245,7 +7299,7 @@ func TestVector100WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector101WireConformanceToRequestResponsesSimple(t *testing.T) { +func TestVector102WireConformanceToRequestResponsesSimple(t *testing.T) { vectorJSON := `{ "name": "responses_simple", "description": "Responses API: system becomes instructions, messages become input items", @@ -7305,7 +7359,7 @@ func TestVector101WireConformanceToRequestResponsesSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector102WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { +func TestVector103WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "responses_structured_output", "description": "Responses API: structured output uses text.format.json_schema", @@ -7393,7 +7447,7 @@ func TestVector102WireConformanceToRequestResponsesStructuredOutput(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector103WireConformanceToRequestResponsesWithTools(t *testing.T) { +func TestVector104WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorJSON := `{ "name": "responses_with_tools", "description": "Responses API: tools use flat format (no nested function key)", @@ -7474,7 +7528,7 @@ func TestVector103WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector104WireConformanceToRequestStructuredOutput(t *testing.T) { +func TestVector105WireConformanceToRequestStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "structured_output", "description": "§7.1.6 — outputs converted to response_format with json_schema, strict=true, additionalProperties=false.", @@ -7558,7 +7612,7 @@ func TestVector104WireConformanceToRequestStructuredOutput(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector105WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { +func TestVector106WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { vectorJSON := `{ "name": "structured_output_nested_optional", "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", @@ -7661,7 +7715,7 @@ func TestVector105WireConformanceToRequestStructuredOutputNestedOptional(t *test vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector106WireConformanceToRequestToolsBindingsStripped(t *testing.T) { +func TestVector107WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorJSON := `{ "name": "tools_bindings_stripped", "description": "§7.1.3 — Parameters listed in bindings MUST be stripped from wire tools (properties AND required).", @@ -7754,7 +7808,7 @@ func TestVector106WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector107WireConformanceToRequestToolsFunctionWire(t *testing.T) { +func TestVector108WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorJSON := `{ "name": "tools_function_wire", "description": "§7.1.3 — FunctionTool projected as OpenAI function definition with JSON Schema parameters.", @@ -7837,7 +7891,7 @@ func TestVector107WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector108WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { +func TestVector109WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorJSON := `{ "name": "tools_null_when_empty", "description": "§7.1.3 — When no tools are defined, the tools key MUST be absent from the request entirely.", @@ -7887,7 +7941,7 @@ func TestVector108WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector109WireConformanceToRequestToolsStrictMode(t *testing.T) { +func TestVector110WireConformanceToRequestToolsStrictMode(t *testing.T) { vectorJSON := `{ "name": "tools_strict_mode", "description": "§7.1.3 — FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.", diff --git a/runtime/go/prompty/model/vectorbridge/bridge.go b/runtime/go/prompty/model/vectorbridge/bridge.go index 6140421cb..555bda37a 100644 --- a/runtime/go/prompty/model/vectorbridge/bridge.go +++ b/runtime/go/prompty/model/vectorbridge/bridge.go @@ -56,6 +56,27 @@ func DiscoveryConformanceMapModel(impl fixtures.DiscoveryConformance) func(rawIn } } +// LiveChatConformanceComplete decodes LiveChatConformance.complete vector input and invokes a +// typed fixtures.LiveChatConformance. Adapt the returned func into your Adapter, e.g.: +// +// VectorAdapters["LiveChatConformance.complete"] = bridged(vectorbridge.LiveChatConformanceComplete(impl)) +func LiveChatConformanceComplete(impl fixtures.LiveChatConformance) func(rawInput any) (any, error) { + return func(rawInput any) (any, error) { + payload, _ := rawInput.(map[string]any) + var input interface{} + { + inputBytes, marshalErr := json.Marshal(payload["input"]) + if marshalErr != nil { + return nil, marshalErr + } + if unmarshalErr := json.Unmarshal(inputBytes, &input); unmarshalErr != nil { + return nil, unmarshalErr + } + } + return impl.Complete(input) + } +} + // LoadConformanceLoad decodes LoadConformance.load vector input and invokes a // typed fixtures.LoadConformance. Adapt the returned func into your Adapter, e.g.: // diff --git a/runtime/go/prompty/vectoradapters/vectoradapters.go b/runtime/go/prompty/vectoradapters/vectoradapters.go index 568487144..617e31ed7 100644 --- a/runtime/go/prompty/vectoradapters/vectoradapters.go +++ b/runtime/go/prompty/vectoradapters/vectoradapters.go @@ -79,7 +79,9 @@ var VectorAdapters = map[string]Adapter{ // VectorWaivers records explicit, honest conformance gaps. Empty: every gap is // being driven to green rather than waived. -var VectorWaivers = map[string]string{} +var VectorWaivers = map[string]string{ + "LiveChatConformance.complete": "drill: consumer-owned transport-seam probe (Python/TS only)", +} // VectorDoubles is reserved for deterministic test doubles. var VectorDoubles = map[string]any{} diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LiveChatConformance.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LiveChatConformance.java new file mode 100644 index 000000000..c243071bc --- /dev/null +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LiveChatConformance.java @@ -0,0 +1,12 @@ +// +// Code generated by Typra emitter; DO NOT EDIT. +package com.microsoft.prompty.model; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public interface LiveChatConformance { + Object complete(Object input); +} diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java index 1056bd1fb..addcba2ad 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java @@ -77,7 +77,11 @@ public static Map adapters() { } public static Map waivers() { - return new LinkedHashMap<>(); + Map waivers = new LinkedHashMap<>(); + waivers.put( + "LiveChatConformance.complete", + "drill: consumer-owned transport-seam probe (Python/TS only)"); + return waivers; } public static Object doubles() { diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java index 81e8f9050..a1d21a6b0 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java @@ -169,658 +169,665 @@ private static void vector15DiscoveryConformanceMapModelOpenaiModelFinetuneNoOwn VectorRunner.runVector("DiscoveryConformance", "mapModel", vector, false, seam()); } - private static void vector16LoadConformanceLoadBasicLoad() throws Exception { + private static void vector16LiveChatConformanceCompleteOpenaiChatDrill() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"openai_chat_drill\",\"stage\":\"drill\",\"provider\":\"openai\",\"input\":{\"provider\":\"openai\",\"model\":\"gpt-4o-mini\",\"apiKey\":{\"$env\":\"OPENAI_API_KEY\"},\"endpoint\":{\"$env\":\"OPENAI_BASE_URL\"},\"messages\":[{\"role\":\"user\",\"content\":\"Say hello in exactly one word.\"}],\"options\":{\"temperature\":0,\"maxOutputTokens\":16}},\"exchange\":{\"transport\":{\"baseUrl\":{\"$env\":\"OPENAI_BASE_URL\"}},\"cassette\":{\"$json\":\"cassettes/openai_chat_drill.json\"},\"planes\":[\"transport\",\"wire\",\"semantic\"]},\"expected\":{\"role\":\"assistant\",\"contentNonEmpty\":true,\"finishReasonInEnum\":true},\"operation\":\"complete\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LiveChatConformance", "complete", vector, false, seam()); + } + + private static void vector17LoadConformanceLoadBasicLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"basic_load\",\"description\":\"Load basic.prompty with env var resolution and verify all PromptAgent fields\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"basic-prompt\",\"description\":\"A basic prompt for testing\",\"metadata\":{\"authors\":[\"testauthor\"]},\"model\":{\"id\":\"gpt-4\",\"provider\":\"openai\",\"apiType\":\"chat\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"},\"options\":{\"temperature\":0.7,\"maxOutputTokens\":1000}},\"inputs\":[{\"name\":\"firstName\",\"kind\":\"string\",\"default\":\"Jane\"},{\"name\":\"lastName\",\"kind\":\"string\",\"default\":\"Doe\"},{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What is the meaning of life?\"}],\"template\":{\"format\":{\"kind\":\"jinja2\"},\"parser\":{\"kind\":\"prompty\"}},\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector17LoadConformanceLoadConnectionTypesLoad() throws Exception { + private static void vector18LoadConformanceLoadConnectionTypesLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector18LoadConformanceLoadEmbeddingLoad() throws Exception { + private static void vector19LoadConformanceLoadEmbeddingLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"embedding_load\",\"description\":\"Load embedding.prompty and verify apiType=embedding with correct model id\",\"stage\":\"load\",\"input\":{\"fixture\":\"embedding.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"embedding\",\"model\":{\"id\":\"text-embedding-3-small\",\"provider\":\"openai\",\"apiType\":\"embedding\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"}},\"inputs\":[{\"name\":\"text\",\"kind\":\"string\",\"default\":\"Hello world\"}],\"instructions\":\"{{text}}\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector19LoadConformanceLoadEmptyFrontmatterBodyOnly() throws Exception { + private static void vector20LoadConformanceLoadEmptyFrontmatterBodyOnly() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"empty_frontmatter_body_only\",\"description\":\"Empty frontmatter with body \\u2014 instructions extracted, defaults applied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"empty-fm\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"empty-fm\",\"model\":null,\"inputs\":null,\"tools\":null},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector20LoadConformanceLoadEnvDefault() throws Exception { + private static void vector21LoadConformanceLoadEnvDefault() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector21LoadConformanceLoadEnvMissingError() throws Exception { + private static void vector22LoadConformanceLoadEnvMissingError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"env_missing_error\",\"description\":\"${env:NONEXISTENT} with no default and var not set raises an error\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-error-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:NONEXISTENT}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"env_var_not_set\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector22LoadConformanceLoadEnvResolution() throws Exception { + private static void vector23LoadConformanceLoadEnvResolution() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"env_resolution\",\"description\":\"${env:MY_VAR} resolves to the environment variable value when set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MY_VAR}\"}}},\"env\":{\"MY_VAR\":\"hello\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"hello\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector23LoadConformanceLoadFileReferenceInTreeAllowed() throws Exception { + private static void vector24LoadConformanceLoadFileReferenceInTreeAllowed() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"file_reference_in_tree_allowed\",\"description\":\"A ${file:sub/data.json} reference that stays within the prompt file's directory tree resolves to the parsed content (containment control alongside the traversal-reject vector)\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-in-tree-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:sub/data.json}\"}},\"files\":{\"sub/data.json\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\",\"apiKey\":\"ok\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector24LoadConformanceLoadFileReferenceTraversalRejected() throws Exception { + private static void vector25LoadConformanceLoadFileReferenceTraversalRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"file_reference_traversal_rejected\",\"description\":\"A ${file:../secret.json} reference that escapes the prompt file's directory via `..` is rejected even though the target file exists, preventing arbitrary file disclosure (GHSA-7pfc-5v9r-j299). Uses forward-slash separators so the vector holds on every OS.\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-traversal-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:../secret.json}\"}},\"files\":{\"../secret.json\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"leaked\"}}},\"expectedError\":{\"kind\":\"file_reference\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector25LoadConformanceLoadFileResolution() throws Exception { + private static void vector26LoadConformanceLoadFileResolution() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"file_resolution\",\"description\":\"${file:shared_connection.json} resolves to the parsed JSON content of the referenced file\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"file-ref-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:shared_connection.json}\"}},\"env\":{\"OPENAI_API_KEY\":\"shared-key\"},\"files\":{\"shared_connection.json\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\",\"apiKey\":\"shared-key\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector26LoadConformanceLoadImageApitypeLoad() throws Exception { + private static void vector27LoadConformanceLoadImageApitypeLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"image_apitype_load\",\"description\":\"An image-generation prompt loads with apiType image and passthrough additionalProperties model options\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}}},\"expected\":{\"kind\":\"prompt\",\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector27LoadConformanceLoadInputScalarShorthand() throws Exception { + private static void vector28LoadConformanceLoadInputScalarShorthand() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"input_scalar_shorthand\",\"description\":\"Scalar input values create typed Property with default set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"scalar-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]}},\"expected\":{\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector28LoadConformanceLoadInputValidationDefaultFill() throws Exception { + private static void vector29LoadConformanceLoadInputValidationDefaultFill() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"input_validation_default_fill\",\"description\":\"An input with a default value is filled in when no value is provided\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"default-fill-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"name\",\"kind\":\"string\",\"default\":\"world\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{\"name\":\"world\"}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector29LoadConformanceLoadInputValidationExampleNotUsed() throws Exception { + private static void vector30LoadConformanceLoadInputValidationExampleNotUsed() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"input_validation_example_not_used\",\"description\":\"An input with example='sample' but no default must NOT use the example at runtime\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"example-not-used-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"required\":false,\"example\":\"sample\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector30LoadConformanceLoadInputValidationOptionalOmit() throws Exception { + private static void vector31LoadConformanceLoadInputValidationOptionalOmit() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"input_validation_optional_omit\",\"description\":\"A non-required input with no default and no value is omitted (not an error)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"optional-omit-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"nickname\",\"kind\":\"string\",\"required\":false}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector31LoadConformanceLoadInputValidationRequired() throws Exception { + private static void vector32LoadConformanceLoadInputValidationRequired() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"input_validation_required\",\"description\":\"A required input with no default and no value provided raises ValueError\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"required-input-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},\"inputs\":{}},\"expectedError\":{\"kind\":\"missing_required_input\",\"field\":\"city\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector32LoadConformanceLoadInstructionsFromBody() throws Exception { + private static void vector33LoadConformanceLoadInstructionsFromBody() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"instructions_from_body\",\"description\":\"The markdown body after the closing --- becomes the instructions field\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector33LoadConformanceLoadInvalidFrontmatterError() throws Exception { + private static void vector34LoadConformanceLoadInvalidFrontmatterError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"invalid_frontmatter_error\",\"description\":\"Malformed YAML in frontmatter raises a parse error\",\"stage\":\"load\",\"input\":{\"frontmatter_raw\":\"---\\nname: [invalid\\n---\\nHello\"},\"expectedError\":{\"kind\":\"invalid_frontmatter\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector34LoadConformanceLoadKindAlwaysPrompt() throws Exception { + private static void vector35LoadConformanceLoadKindAlwaysPrompt() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"kind_always_prompt\",\"description\":\"The loader always injects kind='prompt' \\u2014 .prompty files never specify kind themselves\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector35LoadConformanceLoadMinimalLoad() throws Exception { + private static void vector36LoadConformanceLoadMinimalLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"minimal_load\",\"description\":\"Load minimal.prompty with model shorthand and bare-minimum frontmatter\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"minimal\",\"model\":{\"id\":\"gpt-4\"},\"instructions\":\"system:\\nHello world.\",\"inputs\":null,\"outputs\":null,\"tools\":null},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector36LoadConformanceLoadMissingFileError() throws Exception { + private static void vector37LoadConformanceLoadMissingFileError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"missing_file_error\",\"description\":\"Loading a nonexistent .prompty file raises FileNotFoundError\",\"stage\":\"load\",\"input\":{\"fixture\":\"nonexistent.prompty\"},\"expectedError\":{\"kind\":\"file_not_found\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector37LoadConformanceLoadModelShorthand() throws Exception { + private static void vector38LoadConformanceLoadModelShorthand() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"model_shorthand\",\"description\":\"String shorthand 'model: gpt-4o' expands to a Model object with id set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"test\",\"model\":\"gpt-4o\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"test\",\"model\":{\"id\":\"gpt-4o\"}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector38LoadConformanceLoadStructuredOutputsLoad() throws Exception { + private static void vector39LoadConformanceLoadStructuredOutputsLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_outputs_load\",\"description\":\"Outputs load as a list of properties preserving kind, description, and required flags\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"structured-output\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]}},\"expected\":{\"kind\":\"prompt\",\"name\":\"structured-output\",\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector39LoadConformanceLoadTemplateStringInvalid() throws Exception { + private static void vector40LoadConformanceLoadTemplateStringInvalid() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"template_string_invalid\",\"description\":\"Template as a bare string is not valid v2 \\u2014 must be an object with format/parser\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"shorthand-test\",\"model\":{\"id\":\"gpt-4\"},\"template\":\"jinja2\"}},\"expectedError\":{\"kind\":\"invalid_template\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector40LoadConformanceLoadToolsCustomLoad() throws Exception { + private static void vector41LoadConformanceLoadToolsCustomLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_custom_load\",\"description\":\"Load a prompty with an unknown tool kind \\u2014 falls through to CustomTool\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"custom-tool-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://custom.example.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\"}]},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector41LoadConformanceLoadToolsFunctionLoad() throws Exception { + private static void vector42LoadConformanceLoadToolsFunctionLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_function_load\",\"description\":\"Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings\",\"stage\":\"load\",\"input\":{\"fixture\":\"tools_function.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"function-tools\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather in a given location\",\"strict\":true,\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"description\":\"The city and state, e.g. San Francisco, CA\"},{\"name\":\"unit\",\"kind\":\"string\",\"description\":\"Temperature unit\",\"enumValues\":[\"celsius\",\"fahrenheit\"]}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"inputs\":[{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What's the weather in Paris?\"},{\"name\":\"preferred_unit\",\"kind\":\"string\",\"default\":\"celsius\"}],\"instructions\":\"system:\\nYou are a helpful assistant with access to tools.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector42LoadConformanceLoadToolsMcpLoad() throws Exception { + private static void vector43LoadConformanceLoadToolsMcpLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_mcp_load\",\"description\":\"Load a prompty with an MCP tool \\u2014 kind, serverName, connection preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"mcp-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-mcp\"}}]}},\"expected\":{\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\"}]},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector43LoadConformanceLoadToolsOpenapiLoad() throws Exception { + private static void vector44LoadConformanceLoadToolsOpenapiLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_openapi_load\",\"description\":\"Load a prompty with an OpenAPI tool \\u2014 specification path preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"openapi-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://api.weather.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\"}]},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector44TurnConformanceReplayMaxIterations() throws Exception { + private static void vector45TurnConformanceReplayMaxIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector45TurnConformanceReplayNoTool() throws Exception { + private static void vector46TurnConformanceReplayNoTool() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector46TurnConformanceReplayPermissionDenied() throws Exception { + private static void vector47TurnConformanceReplayPermissionDenied() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector47TurnConformanceReplayToolFailure() throws Exception { + private static void vector48TurnConformanceReplayToolFailure() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector48TurnConformanceReplayToolSuccess() throws Exception { + private static void vector49TurnConformanceReplayToolSuccess() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector49TurnConformanceRunAssistantToolCallsMetadata() throws Exception { + private static void vector50TurnConformanceRunAssistantToolCallsMetadata() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector50TurnConformanceRunAsyncToolFunction() throws Exception { + private static void vector51TurnConformanceRunAsyncToolFunction() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector51TurnConformanceRunBindingsInjected() throws Exception { + private static void vector52TurnConformanceRunBindingsInjected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\\u00b0C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector52TurnConformanceRunCancellationBeforeLlm() throws Exception { + private static void vector53TurnConformanceRunCancellationBeforeLlm() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_before_llm\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector53TurnConformanceRunCancellationBetweenIterations() throws Exception { + private static void vector54TurnConformanceRunCancellationBetweenIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_iterations\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector54TurnConformanceRunCancellationBetweenTools() throws Exception { + private static void vector55TurnConformanceRunCancellationBetweenTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_tools\",\"description\":\"\\u00a713.2 Cancellation \\u2014 LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector55TurnConformanceRunContextNoTrimWhenFits() throws Exception { + private static void vector56TurnConformanceRunContextNoTrimWhenFits() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector56TurnConformanceRunContextPreservesSystemMessages() throws Exception { + private static void vector57TurnConformanceRunContextPreservesSystemMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_preserves_system_messages\",\"description\":\"\\u00a713.3 Context Window \\u2014 Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \\u2014 they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector57TurnConformanceRunContextTrimBasic() throws Exception { + private static void vector58TurnConformanceRunContextTrimBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_trim_basic\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector58TurnConformanceRunEmptyToolResult() throws Exception { + private static void vector59TurnConformanceRunEmptyToolResult() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \\u2014 it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector59TurnConformanceRunEventsBasicToolLoop() throws Exception { + private static void vector60TurnConformanceRunEventsBasicToolLoop() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_basic_tool_loop\",\"description\":\"\\u00a713.1 Events \\u2014 A 2-turn tool-call loop emits the full event lifecycle: status \\u2192 tool_call_start \\u2192 tool_result \\u2192 messages_updated \\u2192 done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\\u00b0F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector60TurnConformanceRunEventsErrorLogged() throws Exception { + private static void vector61TurnConformanceRunEventsErrorLogged() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_error_logged\",\"description\":\"\\u00a713.1 Events \\u0097 A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \\u0097 this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector61TurnConformanceRunEventsNoTools() throws Exception { + private static void vector62TurnConformanceRunEventsNoTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_no_tools\",\"description\":\"\\u00a713.1 Events \\u2014 Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector62TurnConformanceRunGuardrailAllPass() throws Exception { + private static void vector63TurnConformanceRunGuardrailAllPass() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_all_pass\",\"description\":\"\\u00a713.4 Guardrails \\u2014 All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector63TurnConformanceRunGuardrailInputDeny() throws Exception { + private static void vector64TurnConformanceRunGuardrailInputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_input_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector64TurnConformanceRunGuardrailOutputDeny() throws Exception { + private static void vector65TurnConformanceRunGuardrailOutputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_output_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector65TurnConformanceRunGuardrailToolDeny() throws Exception { + private static void vector66TurnConformanceRunGuardrailToolDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_tool_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \\u2014 its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector66TurnConformanceRunMaxIterationsExceeded() throws Exception { + private static void vector67TurnConformanceRunMaxIterationsExceeded() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\\u00b0F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\\u00b0F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\\u00b0F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\\u00b0F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\\u00b0F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\\u00b0F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\\u00b0F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\\u00b0F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\\u00b0F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector67TurnConformanceRunMultiTurnToolCalls() throws Exception { + private static void vector68TurnConformanceRunMultiTurnToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\\u00b0C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector68TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { + private static void vector69TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \\u2014 get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\\u00b0F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector69TurnConformanceRunNoToolCalls() throws Exception { + private static void vector70TurnConformanceRunNoToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \\u2014 the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector70TurnConformanceRunParallelToolsBasic() throws Exception { + private static void vector71TurnConformanceRunParallelToolsBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_basic\",\"description\":\"\\u00a713.6 Parallel Tools \\u2014 LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector71TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { + private static void vector72TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\\u00a713.6 Parallel Tools + \\u00a713.4 Guardrails \\u2014 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector72TurnConformanceRunSingleToolCall() throws Exception { + private static void vector73TurnConformanceRunSingleToolCall() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector73TurnConformanceRunSteeringInjectMessage() throws Exception { + private static void vector74TurnConformanceRunSteeringInjectMessage() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_inject_message\",\"description\":\"\\u00a713.5 Steering \\u2014 After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector74TurnConformanceRunSteeringMultipleMessages() throws Exception { + private static void vector75TurnConformanceRunSteeringMultipleMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_multiple_messages\",\"description\":\"\\u00a713.5 Steering \\u2014 Two steering messages are queued and both injected before iteration 2. Tests atomic drain \\u2014 both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\\u00b0F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector75TurnConformanceRunToolNotRegisteredError() throws Exception { + private static void vector76TurnConformanceRunToolNotRegisteredError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector76TurnConformanceRunToolResultMessageFormat() throws Exception { + private static void vector77TurnConformanceRunToolResultMessageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\\u00b0F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\\u00b0F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector77TurnConformanceRunTurnCancelBeforeContext() throws Exception { + private static void vector78TurnConformanceRunTurnCancelBeforeContext() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector78TurnConformanceRunTurnDelegatedProviderState() throws Exception { + private static void vector79TurnConformanceRunTurnDelegatedProviderState() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector79TurnConformanceRunTurnFinalOutput() throws Exception { + private static void vector80TurnConformanceRunTurnFinalOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector80TurnConformanceRunTurnOrderedToolRound() throws Exception { + private static void vector81TurnConformanceRunTurnOrderedToolRound() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector81TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { + private static void vector82TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector82WireConformanceToRequestAnthropicImageFormat() throws Exception { + private static void vector83WireConformanceToRequestAnthropicImageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_image_format\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector83WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { + private static void vector84WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector84WireConformanceToRequestAnthropicOptions() throws Exception { + private static void vector85WireConformanceToRequestAnthropicOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector85WireConformanceToRequestAnthropicSystemSeparate() throws Exception { + private static void vector86WireConformanceToRequestAnthropicSystemSeparate() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_system_separate\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector86WireConformanceToRequestAnthropicToolWire() throws Exception { + private static void vector87WireConformanceToRequestAnthropicToolWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector87WireConformanceToRequestChatAudioMp3() throws Exception { + private static void vector88WireConformanceToRequestChatAudioMp3() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_mp3\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector88WireConformanceToRequestChatAudioPart() throws Exception { + private static void vector89WireConformanceToRequestChatAudioPart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_part\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector89WireConformanceToRequestChatImageBase64() throws Exception { + private static void vector90WireConformanceToRequestChatImageBase64() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector90WireConformanceToRequestChatImagePart() throws Exception { + private static void vector91WireConformanceToRequestChatImagePart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector91WireConformanceToRequestChatMultipartContent() throws Exception { + private static void vector92WireConformanceToRequestChatMultipartContent() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_multipart_content\",\"description\":\"\\u00a77.1.1/\\u00a77.1.2 \\u2014 Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector92WireConformanceToRequestChatSimple() throws Exception { + private static void vector93WireConformanceToRequestChatSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_simple\",\"description\":\"\\u00a77.1.7 \\u2014 Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector93WireConformanceToRequestChatSingleTextOptimized() throws Exception { + private static void vector94WireConformanceToRequestChatSingleTextOptimized() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_single_text_optimized\",\"description\":\"\\u00a77.1.1 \\u2014 Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector94WireConformanceToRequestChatWithOptions() throws Exception { + private static void vector95WireConformanceToRequestChatWithOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_with_options\",\"description\":\"\\u00a77.1.5 \\u2014 All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector95WireConformanceToRequestEmbeddingWire() throws Exception { + private static void vector96WireConformanceToRequestEmbeddingWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"embedding_wire\",\"description\":\"\\u00a77.2 \\u2014 Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector96WireConformanceToRequestImageWire() throws Exception { + private static void vector97WireConformanceToRequestImageWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"image_wire\",\"description\":\"\\u00a77.3 \\u2014 Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector97WireConformanceToRequestKindToJsonTypeMapping() throws Exception { + private static void vector98WireConformanceToRequestKindToJsonTypeMapping() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\\u00a77.1.4 \\u2014 All Property kind values mapped to JSON Schema types: string\\u2192string, integer\\u2192integer, float\\u2192number, boolean\\u2192boolean, array\\u2192array, object\\u2192object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector98WireConformanceToRequestOptionsAdditionalProperties() throws Exception { + private static void vector99WireConformanceToRequestOptionsAdditionalProperties() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_additional_properties\",\"description\":\"\\u00a77.1.5 \\u2014 additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector99WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { + private static void vector100WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_max_completion_tokens\",\"description\":\"\\u00a77.1.5 \\u2014 maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector100WireConformanceToRequestOptionsStopSequences() throws Exception { + private static void vector101WireConformanceToRequestOptionsStopSequences() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_stop_sequences\",\"description\":\"\\u00a77.1.5 \\u2014 stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector101WireConformanceToRequestResponsesSimple() throws Exception { + private static void vector102WireConformanceToRequestResponsesSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector102WireConformanceToRequestResponsesStructuredOutput() throws Exception { + private static void vector103WireConformanceToRequestResponsesStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector103WireConformanceToRequestResponsesWithTools() throws Exception { + private static void vector104WireConformanceToRequestResponsesWithTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector104WireConformanceToRequestStructuredOutput() throws Exception { + private static void vector105WireConformanceToRequestStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output\",\"description\":\"\\u00a77.1.6 \\u2014 outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector105WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { + private static void vector106WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output_nested_optional\",\"description\":\"\\u00a77.1.4/\\u00a77.1.6 \\u2014 OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector106WireConformanceToRequestToolsBindingsStripped() throws Exception { + private static void vector107WireConformanceToRequestToolsBindingsStripped() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_bindings_stripped\",\"description\":\"\\u00a77.1.3 \\u2014 Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector107WireConformanceToRequestToolsFunctionWire() throws Exception { + private static void vector108WireConformanceToRequestToolsFunctionWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_function_wire\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector108WireConformanceToRequestToolsNullWhenEmpty() throws Exception { + private static void vector109WireConformanceToRequestToolsNullWhenEmpty() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_null_when_empty\",\"description\":\"\\u00a77.1.3 \\u2014 When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector109WireConformanceToRequestToolsStrictMode() throws Exception { + private static void vector110WireConformanceToRequestToolsStrictMode() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_strict_mode\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); @@ -845,100 +852,101 @@ public static void run() { failed += runCase("DiscoveryConformance.mapModel:foundry_deployment_nested_arm", VectorConformanceTests::vector13DiscoveryConformanceMapModelFoundryDeploymentNestedArm); failed += runCase("DiscoveryConformance.mapModel:openai_model_basic", VectorConformanceTests::vector14DiscoveryConformanceMapModelOpenaiModelBasic); failed += runCase("DiscoveryConformance.mapModel:openai_model_finetune_no_owner", VectorConformanceTests::vector15DiscoveryConformanceMapModelOpenaiModelFinetuneNoOwner); - failed += runCase("LoadConformance.load:basic_load", VectorConformanceTests::vector16LoadConformanceLoadBasicLoad); - failed += runCase("LoadConformance.load:connection_types_load", VectorConformanceTests::vector17LoadConformanceLoadConnectionTypesLoad); - failed += runCase("LoadConformance.load:embedding_load", VectorConformanceTests::vector18LoadConformanceLoadEmbeddingLoad); - failed += runCase("LoadConformance.load:empty_frontmatter_body_only", VectorConformanceTests::vector19LoadConformanceLoadEmptyFrontmatterBodyOnly); - failed += runCase("LoadConformance.load:env_default", VectorConformanceTests::vector20LoadConformanceLoadEnvDefault); - failed += runCase("LoadConformance.load:env_missing_error", VectorConformanceTests::vector21LoadConformanceLoadEnvMissingError); - failed += runCase("LoadConformance.load:env_resolution", VectorConformanceTests::vector22LoadConformanceLoadEnvResolution); - failed += runCase("LoadConformance.load:file_reference_in_tree_allowed", VectorConformanceTests::vector23LoadConformanceLoadFileReferenceInTreeAllowed); - failed += runCase("LoadConformance.load:file_reference_traversal_rejected", VectorConformanceTests::vector24LoadConformanceLoadFileReferenceTraversalRejected); - failed += runCase("LoadConformance.load:file_resolution", VectorConformanceTests::vector25LoadConformanceLoadFileResolution); - failed += runCase("LoadConformance.load:image_apitype_load", VectorConformanceTests::vector26LoadConformanceLoadImageApitypeLoad); - failed += runCase("LoadConformance.load:input_scalar_shorthand", VectorConformanceTests::vector27LoadConformanceLoadInputScalarShorthand); - failed += runCase("LoadConformance.load:input_validation_default_fill", VectorConformanceTests::vector28LoadConformanceLoadInputValidationDefaultFill); - failed += runCase("LoadConformance.load:input_validation_example_not_used", VectorConformanceTests::vector29LoadConformanceLoadInputValidationExampleNotUsed); - failed += runCase("LoadConformance.load:input_validation_optional_omit", VectorConformanceTests::vector30LoadConformanceLoadInputValidationOptionalOmit); - failed += runCase("LoadConformance.load:input_validation_required", VectorConformanceTests::vector31LoadConformanceLoadInputValidationRequired); - failed += runCase("LoadConformance.load:instructions_from_body", VectorConformanceTests::vector32LoadConformanceLoadInstructionsFromBody); - failed += runCase("LoadConformance.load:invalid_frontmatter_error", VectorConformanceTests::vector33LoadConformanceLoadInvalidFrontmatterError); - failed += runCase("LoadConformance.load:kind_always_prompt", VectorConformanceTests::vector34LoadConformanceLoadKindAlwaysPrompt); - failed += runCase("LoadConformance.load:minimal_load", VectorConformanceTests::vector35LoadConformanceLoadMinimalLoad); - failed += runCase("LoadConformance.load:missing_file_error", VectorConformanceTests::vector36LoadConformanceLoadMissingFileError); - failed += runCase("LoadConformance.load:model_shorthand", VectorConformanceTests::vector37LoadConformanceLoadModelShorthand); - failed += runCase("LoadConformance.load:structured_outputs_load", VectorConformanceTests::vector38LoadConformanceLoadStructuredOutputsLoad); - failed += runCase("LoadConformance.load:template_string_invalid", VectorConformanceTests::vector39LoadConformanceLoadTemplateStringInvalid); - failed += runCase("LoadConformance.load:tools_custom_load", VectorConformanceTests::vector40LoadConformanceLoadToolsCustomLoad); - failed += runCase("LoadConformance.load:tools_function_load", VectorConformanceTests::vector41LoadConformanceLoadToolsFunctionLoad); - failed += runCase("LoadConformance.load:tools_mcp_load", VectorConformanceTests::vector42LoadConformanceLoadToolsMcpLoad); - failed += runCase("LoadConformance.load:tools_openapi_load", VectorConformanceTests::vector43LoadConformanceLoadToolsOpenapiLoad); - failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector44TurnConformanceReplayMaxIterations); - failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector45TurnConformanceReplayNoTool); - failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector46TurnConformanceReplayPermissionDenied); - failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector47TurnConformanceReplayToolFailure); - failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector48TurnConformanceReplayToolSuccess); - failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector49TurnConformanceRunAssistantToolCallsMetadata); - failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector50TurnConformanceRunAsyncToolFunction); - failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector51TurnConformanceRunBindingsInjected); - failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector52TurnConformanceRunCancellationBeforeLlm); - failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector53TurnConformanceRunCancellationBetweenIterations); - failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector54TurnConformanceRunCancellationBetweenTools); - failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector55TurnConformanceRunContextNoTrimWhenFits); - failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector56TurnConformanceRunContextPreservesSystemMessages); - failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector57TurnConformanceRunContextTrimBasic); - failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector58TurnConformanceRunEmptyToolResult); - failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector59TurnConformanceRunEventsBasicToolLoop); - failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector60TurnConformanceRunEventsErrorLogged); - failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector61TurnConformanceRunEventsNoTools); - failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector62TurnConformanceRunGuardrailAllPass); - failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector63TurnConformanceRunGuardrailInputDeny); - failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector64TurnConformanceRunGuardrailOutputDeny); - failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector65TurnConformanceRunGuardrailToolDeny); - failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector66TurnConformanceRunMaxIterationsExceeded); - failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector67TurnConformanceRunMultiTurnToolCalls); - failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector68TurnConformanceRunMultipleToolCallsSingleTurn); - failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector69TurnConformanceRunNoToolCalls); - failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector70TurnConformanceRunParallelToolsBasic); - failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector71TurnConformanceRunParallelToolsWithGuardrailDeny); - failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector72TurnConformanceRunSingleToolCall); - failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector73TurnConformanceRunSteeringInjectMessage); - failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector74TurnConformanceRunSteeringMultipleMessages); - failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector75TurnConformanceRunToolNotRegisteredError); - failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector76TurnConformanceRunToolResultMessageFormat); - failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector77TurnConformanceRunTurnCancelBeforeContext); - failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector78TurnConformanceRunTurnDelegatedProviderState); - failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector79TurnConformanceRunTurnFinalOutput); - failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector80TurnConformanceRunTurnOrderedToolRound); - failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector81TurnConformanceRunTurnPermissionDenialIsModelVisible); - failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector82WireConformanceToRequestAnthropicImageFormat); - failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector83WireConformanceToRequestAnthropicMaxTokensRequired); - failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector84WireConformanceToRequestAnthropicOptions); - failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector85WireConformanceToRequestAnthropicSystemSeparate); - failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector86WireConformanceToRequestAnthropicToolWire); - failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector87WireConformanceToRequestChatAudioMp3); - failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector88WireConformanceToRequestChatAudioPart); - failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector89WireConformanceToRequestChatImageBase64); - failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector90WireConformanceToRequestChatImagePart); - failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector91WireConformanceToRequestChatMultipartContent); - failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector92WireConformanceToRequestChatSimple); - failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector93WireConformanceToRequestChatSingleTextOptimized); - failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector94WireConformanceToRequestChatWithOptions); - failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector95WireConformanceToRequestEmbeddingWire); - failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector96WireConformanceToRequestImageWire); - failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector97WireConformanceToRequestKindToJsonTypeMapping); - failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector98WireConformanceToRequestOptionsAdditionalProperties); - failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector99WireConformanceToRequestOptionsMaxCompletionTokens); - failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector100WireConformanceToRequestOptionsStopSequences); - failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector101WireConformanceToRequestResponsesSimple); - failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector102WireConformanceToRequestResponsesStructuredOutput); - failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector103WireConformanceToRequestResponsesWithTools); - failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector104WireConformanceToRequestStructuredOutput); - failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector105WireConformanceToRequestStructuredOutputNestedOptional); - failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector106WireConformanceToRequestToolsBindingsStripped); - failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector107WireConformanceToRequestToolsFunctionWire); - failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector108WireConformanceToRequestToolsNullWhenEmpty); - failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector109WireConformanceToRequestToolsStrictMode); + failed += runCase("LiveChatConformance.complete:openai_chat_drill", VectorConformanceTests::vector16LiveChatConformanceCompleteOpenaiChatDrill); + failed += runCase("LoadConformance.load:basic_load", VectorConformanceTests::vector17LoadConformanceLoadBasicLoad); + failed += runCase("LoadConformance.load:connection_types_load", VectorConformanceTests::vector18LoadConformanceLoadConnectionTypesLoad); + failed += runCase("LoadConformance.load:embedding_load", VectorConformanceTests::vector19LoadConformanceLoadEmbeddingLoad); + failed += runCase("LoadConformance.load:empty_frontmatter_body_only", VectorConformanceTests::vector20LoadConformanceLoadEmptyFrontmatterBodyOnly); + failed += runCase("LoadConformance.load:env_default", VectorConformanceTests::vector21LoadConformanceLoadEnvDefault); + failed += runCase("LoadConformance.load:env_missing_error", VectorConformanceTests::vector22LoadConformanceLoadEnvMissingError); + failed += runCase("LoadConformance.load:env_resolution", VectorConformanceTests::vector23LoadConformanceLoadEnvResolution); + failed += runCase("LoadConformance.load:file_reference_in_tree_allowed", VectorConformanceTests::vector24LoadConformanceLoadFileReferenceInTreeAllowed); + failed += runCase("LoadConformance.load:file_reference_traversal_rejected", VectorConformanceTests::vector25LoadConformanceLoadFileReferenceTraversalRejected); + failed += runCase("LoadConformance.load:file_resolution", VectorConformanceTests::vector26LoadConformanceLoadFileResolution); + failed += runCase("LoadConformance.load:image_apitype_load", VectorConformanceTests::vector27LoadConformanceLoadImageApitypeLoad); + failed += runCase("LoadConformance.load:input_scalar_shorthand", VectorConformanceTests::vector28LoadConformanceLoadInputScalarShorthand); + failed += runCase("LoadConformance.load:input_validation_default_fill", VectorConformanceTests::vector29LoadConformanceLoadInputValidationDefaultFill); + failed += runCase("LoadConformance.load:input_validation_example_not_used", VectorConformanceTests::vector30LoadConformanceLoadInputValidationExampleNotUsed); + failed += runCase("LoadConformance.load:input_validation_optional_omit", VectorConformanceTests::vector31LoadConformanceLoadInputValidationOptionalOmit); + failed += runCase("LoadConformance.load:input_validation_required", VectorConformanceTests::vector32LoadConformanceLoadInputValidationRequired); + failed += runCase("LoadConformance.load:instructions_from_body", VectorConformanceTests::vector33LoadConformanceLoadInstructionsFromBody); + failed += runCase("LoadConformance.load:invalid_frontmatter_error", VectorConformanceTests::vector34LoadConformanceLoadInvalidFrontmatterError); + failed += runCase("LoadConformance.load:kind_always_prompt", VectorConformanceTests::vector35LoadConformanceLoadKindAlwaysPrompt); + failed += runCase("LoadConformance.load:minimal_load", VectorConformanceTests::vector36LoadConformanceLoadMinimalLoad); + failed += runCase("LoadConformance.load:missing_file_error", VectorConformanceTests::vector37LoadConformanceLoadMissingFileError); + failed += runCase("LoadConformance.load:model_shorthand", VectorConformanceTests::vector38LoadConformanceLoadModelShorthand); + failed += runCase("LoadConformance.load:structured_outputs_load", VectorConformanceTests::vector39LoadConformanceLoadStructuredOutputsLoad); + failed += runCase("LoadConformance.load:template_string_invalid", VectorConformanceTests::vector40LoadConformanceLoadTemplateStringInvalid); + failed += runCase("LoadConformance.load:tools_custom_load", VectorConformanceTests::vector41LoadConformanceLoadToolsCustomLoad); + failed += runCase("LoadConformance.load:tools_function_load", VectorConformanceTests::vector42LoadConformanceLoadToolsFunctionLoad); + failed += runCase("LoadConformance.load:tools_mcp_load", VectorConformanceTests::vector43LoadConformanceLoadToolsMcpLoad); + failed += runCase("LoadConformance.load:tools_openapi_load", VectorConformanceTests::vector44LoadConformanceLoadToolsOpenapiLoad); + failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector45TurnConformanceReplayMaxIterations); + failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector46TurnConformanceReplayNoTool); + failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector47TurnConformanceReplayPermissionDenied); + failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector48TurnConformanceReplayToolFailure); + failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector49TurnConformanceReplayToolSuccess); + failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector50TurnConformanceRunAssistantToolCallsMetadata); + failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector51TurnConformanceRunAsyncToolFunction); + failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector52TurnConformanceRunBindingsInjected); + failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector53TurnConformanceRunCancellationBeforeLlm); + failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector54TurnConformanceRunCancellationBetweenIterations); + failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector55TurnConformanceRunCancellationBetweenTools); + failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector56TurnConformanceRunContextNoTrimWhenFits); + failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector57TurnConformanceRunContextPreservesSystemMessages); + failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector58TurnConformanceRunContextTrimBasic); + failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector59TurnConformanceRunEmptyToolResult); + failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector60TurnConformanceRunEventsBasicToolLoop); + failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector61TurnConformanceRunEventsErrorLogged); + failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector62TurnConformanceRunEventsNoTools); + failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector63TurnConformanceRunGuardrailAllPass); + failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector64TurnConformanceRunGuardrailInputDeny); + failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector65TurnConformanceRunGuardrailOutputDeny); + failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector66TurnConformanceRunGuardrailToolDeny); + failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector67TurnConformanceRunMaxIterationsExceeded); + failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector68TurnConformanceRunMultiTurnToolCalls); + failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector69TurnConformanceRunMultipleToolCallsSingleTurn); + failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector70TurnConformanceRunNoToolCalls); + failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector71TurnConformanceRunParallelToolsBasic); + failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector72TurnConformanceRunParallelToolsWithGuardrailDeny); + failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector73TurnConformanceRunSingleToolCall); + failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector74TurnConformanceRunSteeringInjectMessage); + failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector75TurnConformanceRunSteeringMultipleMessages); + failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector76TurnConformanceRunToolNotRegisteredError); + failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector77TurnConformanceRunToolResultMessageFormat); + failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector78TurnConformanceRunTurnCancelBeforeContext); + failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector79TurnConformanceRunTurnDelegatedProviderState); + failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector80TurnConformanceRunTurnFinalOutput); + failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector81TurnConformanceRunTurnOrderedToolRound); + failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector82TurnConformanceRunTurnPermissionDenialIsModelVisible); + failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector83WireConformanceToRequestAnthropicImageFormat); + failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector84WireConformanceToRequestAnthropicMaxTokensRequired); + failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector85WireConformanceToRequestAnthropicOptions); + failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector86WireConformanceToRequestAnthropicSystemSeparate); + failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector87WireConformanceToRequestAnthropicToolWire); + failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector88WireConformanceToRequestChatAudioMp3); + failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector89WireConformanceToRequestChatAudioPart); + failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector90WireConformanceToRequestChatImageBase64); + failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector91WireConformanceToRequestChatImagePart); + failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector92WireConformanceToRequestChatMultipartContent); + failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector93WireConformanceToRequestChatSimple); + failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector94WireConformanceToRequestChatSingleTextOptimized); + failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector95WireConformanceToRequestChatWithOptions); + failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector96WireConformanceToRequestEmbeddingWire); + failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector97WireConformanceToRequestImageWire); + failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector98WireConformanceToRequestKindToJsonTypeMapping); + failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector99WireConformanceToRequestOptionsAdditionalProperties); + failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector100WireConformanceToRequestOptionsMaxCompletionTokens); + failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector101WireConformanceToRequestOptionsStopSequences); + failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector102WireConformanceToRequestResponsesSimple); + failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector103WireConformanceToRequestResponsesStructuredOutput); + failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector104WireConformanceToRequestResponsesWithTools); + failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector105WireConformanceToRequestStructuredOutput); + failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector106WireConformanceToRequestStructuredOutputNestedOptional); + failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector107WireConformanceToRequestToolsBindingsStripped); + failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector108WireConformanceToRequestToolsFunctionWire); + failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector109WireConformanceToRequestToolsNullWhenEmpty); + failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector110WireConformanceToRequestToolsStrictMode); if (failed > 0) { throw new AssertionError(failed + " @vector conformance failure(s)"); } diff --git a/runtime/python/prompty/prompty/model/_LiveChatConformance.py b/runtime/python/prompty/prompty/model/_LiveChatConformance.py new file mode 100644 index 000000000..6ed253b6f --- /dev/null +++ b/runtime/python/prompty/prompty/model/_LiveChatConformance.py @@ -0,0 +1,28 @@ +# +########################################## +# WARNING: This is an auto-generated file. +# DO NOT EDIT THIS FILE DIRECTLY +# ANY EDITS WILL BE LOST +########################################## + +from typing import Any, Protocol, runtime_checkable + + +@runtime_checkable +class LiveChatConformance(Protocol): + """Conformance-only seam hosting the "service drill" vectors: a conformance + vector that crosses the REAL Executor->Processor transport seam instead of + stopping at the SDK edge. A drill runs against a live provider when + credentials are present and REPLAYS a recorded cassette when they are not; + the live<->replay parity is the gate. + + Like every other conformance seam this interface exists purely to carry + """ + + def complete(self, input: Any) -> Any: + """Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes""" + raise NotImplementedError + + async def complete_async(self, input: Any) -> Any: + """Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes (async variant)""" + raise NotImplementedError diff --git a/runtime/python/prompty/prompty/model/__init__.py b/runtime/python/prompty/prompty/model/__init__.py index a82eda84c..e13bc5b02 100644 --- a/runtime/python/prompty/prompty/model/__init__.py +++ b/runtime/python/prompty/prompty/model/__init__.py @@ -16,6 +16,7 @@ from ._EventSink import EventSink from ._Executor import Executor from ._HostToolExecutor import HostToolExecutor +from ._LiveChatConformance import LiveChatConformance from ._LoadConformance import LoadConformance from ._ModelLister import ModelLister from ._Parser import Parser @@ -379,6 +380,7 @@ "EventSink", "Executor", "HostToolExecutor", + "LiveChatConformance", "LoadConformance", "ModelLister", "Parser", diff --git a/runtime/python/prompty/tests/model/_drill_replay.py b/runtime/python/prompty/tests/model/_drill_replay.py new file mode 100644 index 000000000..8075c4016 --- /dev/null +++ b/runtime/python/prompty/tests/model/_drill_replay.py @@ -0,0 +1,97 @@ +"""Local cassette-replay HTTP transport for the service-drill vector. + +The drill binds the Executor->Processor transport seam via **base-URL redirect**: +the provider SDK is pointed at a local ``http.server`` thread that serves a +recorded response and records the inbound request. This is the consumer-owned +"roster" half of the drill -- deliberately NOT modeled in TypeSpec/Typra, since +cassette bytes and provider-SDK transport binding stay runtime-owned. + +Two use sites share one server: + +* **capture** (tooling, `record_cassette.py`): serve a synthesized provider + response, drive the real executor once, and freeze the captured request as the + cassette's ``request`` plane. +* **replay** (`vector_adapters.py` drill adapter): serve the cassette's recorded + response and assert the freshly captured request matches the cassette + (``wire`` plane). + +No production code changes: the executor's own connection ``endpoint`` is set to +``server.base_url`` and its OpenAI SDK POSTs there exactly as it would to the +real base URL. +""" + +from __future__ import annotations + +import http.server +import json +import threading +from typing import Any + + +class _RecordingHandler(http.server.BaseHTTPRequestHandler): + """Records each POST body and replies with the server's canned response.""" + + # Silence the default stderr access log so conformance output stays clean. + def log_message(self, *args: Any) -> None: # noqa: D401 - stdlib override + return + + def do_POST(self) -> None: # noqa: N802 - stdlib handler name + length = int(self.headers.get("Content-Length", "0") or "0") + raw = self.rfile.read(length) if length else b"" + try: + body: Any = json.loads(raw) if raw else {} + except json.JSONDecodeError: + body = {"__raw__": raw.decode("utf-8", "replace")} + server: Any = self.server + server.captured.append({"method": "POST", "path": self.path, "body": body}) + + payload = json.dumps(server.response_body).encode("utf-8") + self.send_response(server.response_status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + +class CassetteReplayServer: + """A background HTTP server that replays one response and records requests. + + Use as a context manager. ``base_url`` is the redirect target to hand the + provider SDK; ``last_request`` is the most recently captured outbound request + (the wire plane's observed value). + """ + + def __init__(self, response_body: Any, response_status: int = 200) -> None: + self._response_body = response_body + self._response_status = response_status + self._httpd: http.server.HTTPServer | None = None + self._thread: threading.Thread | None = None + self.captured: list[dict[str, Any]] = [] + + def __enter__(self) -> CassetteReplayServer: + httpd = http.server.HTTPServer(("127.0.0.1", 0), _RecordingHandler) + # Stash server-scoped state the handler reads off ``self.server``. + httpd.captured = self.captured # type: ignore[attr-defined] + httpd.response_body = self._response_body # type: ignore[attr-defined] + httpd.response_status = self._response_status # type: ignore[attr-defined] + self._httpd = httpd + self._thread = threading.Thread(target=httpd.serve_forever, daemon=True) + self._thread.start() + return self + + def __exit__(self, *exc: Any) -> None: + if self._httpd is not None: + self._httpd.shutdown() + self._httpd.server_close() + if self._thread is not None: + self._thread.join(timeout=2) + + @property + def base_url(self) -> str: + assert self._httpd is not None, "server not started" + host, port = self._httpd.server_address[:2] + return f"http://{host}:{port}" + + @property + def last_request(self) -> dict[str, Any] | None: + return self.captured[-1] if self.captured else None diff --git a/runtime/python/prompty/tests/model/cassettes/openai_chat_drill.json b/runtime/python/prompty/tests/model/cassettes/openai_chat_drill.json new file mode 100644 index 000000000..8afdc52e0 --- /dev/null +++ b/runtime/python/prompty/tests/model/cassettes/openai_chat_drill.json @@ -0,0 +1,44 @@ +{ + "_comment": "Service-drill cassette for LiveChatConformance.complete/openai_chat_drill. Consumer-owned roster: NOT modeled in TypeSpec/Typra. Regenerate with `uv run python tests/model/record_cassette.py`. `request.body` is the wire plane the drill adapter asserts; `response.body` is replayed as the seam bytes.", + "recordedAt": "2026-09-04T22:41:20Z", + "request": { + "body": { + "max_completion_tokens": 16, + "messages": [ + { + "content": "Say hello in exactly one word.", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "temperature": 0 + }, + "method": "POST", + "path": "/chat/completions" + }, + "response": { + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "Hello", + "role": "assistant" + } + } + ], + "created": 1700000000, + "id": "chatcmpl-drill-openai-0001", + "model": "gpt-4o-mini", + "object": "chat.completion", + "usage": { + "completion_tokens": 1, + "prompt_tokens": 12, + "total_tokens": 13 + } + }, + "status": 200 + } +} diff --git a/runtime/python/prompty/tests/model/record_cassette.py b/runtime/python/prompty/tests/model/record_cassette.py new file mode 100644 index 000000000..a2572573f --- /dev/null +++ b/runtime/python/prompty/tests/model/record_cassette.py @@ -0,0 +1,107 @@ +"""Cassette-capture tooling for the service-drill vector (NOT a pytest test). + +Run this to (re)generate ``cassettes/openai_chat_drill.json``. It synthesizes a +schema-valid OpenAI ``ChatCompletion`` response, serves it from a local +``CassetteReplayServer``, drives the REAL executor once through the same code +path the drill adapter uses (``_drill_execute``), and freezes the captured +outbound request as the cassette's ``request`` plane. The recorded response is +the ``response`` plane. + +This lives in CI/dev tooling, NEVER in the assertion path: it does not diff or +auto-rewrite during conformance. The ``request`` it records is exactly what the +drill's wire plane later asserts against, so capturing (rather than +hand-fabricating) keeps the wire contract honest and reproducible. + +Usage (from ``runtime/python/prompty``): + + uv run python tests/model/record_cassette.py + +The prefix is deliberately ``record_`` (not ``test_``) so pytest ignores it. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + +from _drill_replay import CassetteReplayServer # noqa: E402 +from vector_adapters import _DRILL_REPLAY_API_KEY, _drill_execute # noqa: E402 + +# Mirror the drill vector's `input` (schema/model/conformance/vectors/drill.tsp). +# The apiKey/endpoint refs are intentionally absent here -- capture always binds +# the transport to the local replay server, never a real provider. +_DRILL_INPUT: dict = { + "provider": "openai", + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "Say hello in exactly one word."}], + "options": {"temperature": 0, "maxOutputTokens": 16}, +} + + +def _synthesize_response() -> dict: + """A schema-valid OpenAI ChatCompletion the SDK can parse into a response.""" + return { + "id": "chatcmpl-drill-openai-0001", + "object": "chat.completion", + "created": 1_700_000_000, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + "logprobs": None, + } + ], + "usage": {"prompt_tokens": 12, "completion_tokens": 1, "total_tokens": 13}, + } + + +def main() -> None: + response_body = _synthesize_response() + + with CassetteReplayServer(response_body) as server: + raw = _drill_execute(_DRILL_INPUT, endpoint=server.base_url, api_key=_DRILL_REPLAY_API_KEY) + if raw is None: + raise SystemExit("capture failed: executor returned no response") + captured = server.last_request + if captured is None: + raise SystemExit("capture failed: no request reached the replay server") + + cassette = { + "_comment": ( + "Service-drill cassette for LiveChatConformance.complete/openai_chat_drill. " + "Consumer-owned roster: NOT modeled in TypeSpec/Typra. Regenerate with " + "`uv run python tests/model/record_cassette.py`. `request.body` is the wire " + "plane the drill adapter asserts; `response.body` is replayed as the seam bytes." + ), + "recordedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "request": { + "method": captured.get("method"), + "path": captured.get("path"), + "body": captured.get("body"), + }, + "response": { + "status": 200, + "body": response_body, + }, + } + + out_dir = _HERE / "cassettes" + out_dir.mkdir(exist_ok=True) + out_path = out_dir / "openai_chat_drill.json" + out_path.write_text(json.dumps(cassette, indent=2, sort_keys=True) + "\n", encoding="utf-8") + rel = os.path.relpath(out_path, Path.cwd()) + print(f"wrote {rel}") + print(f" request.body = {json.dumps(cassette['request']['body'], sort_keys=True)}") + + +if __name__ == "__main__": + main() diff --git a/runtime/python/prompty/tests/model/test_capability_guard_reference.py b/runtime/python/prompty/tests/model/test_capability_guard_reference.py index fad9eebd4..97e89487d 100644 --- a/runtime/python/prompty/tests/model/test_capability_guard_reference.py +++ b/runtime/python/prompty/tests/model/test_capability_guard_reference.py @@ -6,7 +6,8 @@ * the ``VECTOR_CAPABILITIES`` predicates in ``vector_adapters`` (present/absent), * a faithful LOCAL REPLICA of the emitter's two-pass guard so the skip/hard-fail semantics are verified without waiting for a regenerated harness, and -* the structure-not-content ``_live_chat_normalize`` projection. +* the structure-not-content ``_drill_normalize`` projection (the drill's + semantic-plane reducer). The generated harness (``test_vector_conformance.py``) only invokes the guard once a schema vector declares ``requires`` AND the emitter pin is bumped to @@ -207,7 +208,7 @@ def _openai_response(role: str = "assistant", content: str = "Hello", finish: st class TestLiveChatNormalize: def test_openai_shape_reduces_to_structure(self) -> None: - result = va._live_chat_normalize(_openai_response(), _context()) + result = va._drill_normalize(_openai_response(), _context()) assert result == { "role": "assistant", "contentNonEmpty": True, @@ -215,11 +216,11 @@ def test_openai_shape_reduces_to_structure(self) -> None: } def test_empty_content_flags_false(self) -> None: - result = va._live_chat_normalize(_openai_response(content=" "), _context()) + result = va._drill_normalize(_openai_response(content=" "), _context()) assert result["contentNonEmpty"] is False def test_finish_reason_outside_enum_flags_false(self) -> None: - result = va._live_chat_normalize(_openai_response(finish="explode"), _context()) + result = va._drill_normalize(_openai_response(finish="explode"), _context()) assert result["finishReasonInEnum"] is False @pytest.mark.parametrize( @@ -227,7 +228,7 @@ def test_finish_reason_outside_enum_flags_false(self) -> None: ["stop", "length", "tool_calls", "content_filter", "function_call"], ) def test_all_canonical_finish_reasons_pass(self, finish: str) -> None: - result = va._live_chat_normalize(_openai_response(finish=finish), _context()) + result = va._drill_normalize(_openai_response(finish=finish), _context()) assert result["finishReasonInEnum"] is True def test_anthropic_shape_maps_stop_reason(self) -> None: @@ -237,7 +238,7 @@ def test_anthropic_shape_maps_stop_reason(self) -> None: response.role = "assistant" response.content = [block] response.stop_reason = "end_turn" - result = va._live_chat_normalize(response, _context()) + result = va._drill_normalize(response, _context()) assert result == { "role": "assistant", "contentNonEmpty": True, @@ -265,8 +266,8 @@ def test_openai_live_chat_structure(self) -> None: "options": {"temperature": 0, "maxOutputTokens": 16}, } context = _context(vector={"input": resolved}) - observed = va._live_chat_invoke(resolved, context) - structure = va._live_chat_normalize(observed, context) + observed = va._drill_execute(resolved, endpoint=None, api_key=resolved["apiKey"]) + structure = va._drill_normalize(observed, context) assert structure["role"] == "assistant" assert structure["contentNonEmpty"] is True assert structure["finishReasonInEnum"] is True diff --git a/runtime/python/prompty/tests/model/test_vector_conformance.py b/runtime/python/prompty/tests/model/test_vector_conformance.py index 0b8b8158b..6c5592067 100644 --- a/runtime/python/prompty/tests/model/test_vector_conformance.py +++ b/runtime/python/prompty/tests/model/test_vector_conformance.py @@ -537,7 +537,60 @@ async def test_vector_15_discoveryconformance_mapmodel_openai_model_finetune_no_ await run_vector("DiscoveryConformance", "mapModel", vector, False, _SEAM) -async def test_vector_16_loadconformance_load_basic_load(): +async def test_vector_16_livechatconformance_complete_openai_chat_drill(): + vector_json = r""" + { + "name": "openai_chat_drill", + "stage": "drill", + "provider": "openai", + "input": { + "provider": "openai", + "model": "gpt-4o-mini", + "apiKey": { + "$env": "OPENAI_API_KEY" + }, + "endpoint": { + "$env": "OPENAI_BASE_URL" + }, + "messages": [ + { + "role": "user", + "content": "Say hello in exactly one word." + } + ], + "options": { + "temperature": 0, + "maxOutputTokens": 16 + } + }, + "exchange": { + "transport": { + "baseUrl": { + "$env": "OPENAI_BASE_URL" + } + }, + "cassette": { + "$json": "cassettes/openai_chat_drill.json" + }, + "planes": [ + "transport", + "wire", + "semantic" + ] + }, + "expected": { + "role": "assistant", + "contentNonEmpty": true, + "finishReasonInEnum": true + }, + "operation": "complete" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LiveChatConformance", "complete", vector, False, _SEAM) + + +async def test_vector_17_loadconformance_load_basic_load(): vector_json = r""" { "name": "basic_load", @@ -606,7 +659,7 @@ async def test_vector_16_loadconformance_load_basic_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_17_loadconformance_load_connection_types_load(): +async def test_vector_18_loadconformance_load_connection_types_load(): vector_json = r""" { "name": "connection_types_load", @@ -640,7 +693,7 @@ async def test_vector_17_loadconformance_load_connection_types_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_18_loadconformance_load_embedding_load(): +async def test_vector_19_loadconformance_load_embedding_load(): vector_json = r""" { "name": "embedding_load", @@ -681,7 +734,7 @@ async def test_vector_18_loadconformance_load_embedding_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_19_loadconformance_load_empty_frontmatter_body_only(): +async def test_vector_20_loadconformance_load_empty_frontmatter_body_only(): vector_json = r""" { "name": "empty_frontmatter_body_only", @@ -706,7 +759,7 @@ async def test_vector_19_loadconformance_load_empty_frontmatter_body_only(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_20_loadconformance_load_env_default(): +async def test_vector_21_loadconformance_load_env_default(): vector_json = r""" { "name": "env_default", @@ -740,7 +793,7 @@ async def test_vector_20_loadconformance_load_env_default(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_21_loadconformance_load_env_missing_error(): +async def test_vector_22_loadconformance_load_env_missing_error(): vector_json = r""" { "name": "env_missing_error", @@ -769,7 +822,7 @@ async def test_vector_21_loadconformance_load_env_missing_error(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_22_loadconformance_load_env_resolution(): +async def test_vector_23_loadconformance_load_env_resolution(): vector_json = r""" { "name": "env_resolution", @@ -805,7 +858,7 @@ async def test_vector_22_loadconformance_load_env_resolution(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_23_loadconformance_load_file_reference_in_tree_allowed(): +async def test_vector_24_loadconformance_load_file_reference_in_tree_allowed(): vector_json = r""" { "name": "file_reference_in_tree_allowed", @@ -843,7 +896,7 @@ async def test_vector_23_loadconformance_load_file_reference_in_tree_allowed(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_24_loadconformance_load_file_reference_traversal_rejected(): +async def test_vector_25_loadconformance_load_file_reference_traversal_rejected(): vector_json = r""" { "name": "file_reference_traversal_rejected", @@ -876,7 +929,7 @@ async def test_vector_24_loadconformance_load_file_reference_traversal_rejected( await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_25_loadconformance_load_file_resolution(): +async def test_vector_26_loadconformance_load_file_resolution(): vector_json = r""" { "name": "file_resolution", @@ -916,7 +969,7 @@ async def test_vector_25_loadconformance_load_file_resolution(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_26_loadconformance_load_image_apitype_load(): +async def test_vector_27_loadconformance_load_image_apitype_load(): vector_json = r""" { "name": "image_apitype_load", @@ -960,7 +1013,7 @@ async def test_vector_26_loadconformance_load_image_apitype_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_27_loadconformance_load_input_scalar_shorthand(): +async def test_vector_28_loadconformance_load_input_scalar_shorthand(): vector_json = r""" { "name": "input_scalar_shorthand", @@ -1005,7 +1058,7 @@ async def test_vector_27_loadconformance_load_input_scalar_shorthand(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_28_loadconformance_load_input_validation_default_fill(): +async def test_vector_29_loadconformance_load_input_validation_default_fill(): vector_json = r""" { "name": "input_validation_default_fill", @@ -1037,7 +1090,7 @@ async def test_vector_28_loadconformance_load_input_validation_default_fill(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_29_loadconformance_load_input_validation_example_not_used(): +async def test_vector_30_loadconformance_load_input_validation_example_not_used(): vector_json = r""" { "name": "input_validation_example_not_used", @@ -1068,7 +1121,7 @@ async def test_vector_29_loadconformance_load_input_validation_example_not_used( await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_30_loadconformance_load_input_validation_optional_omit(): +async def test_vector_31_loadconformance_load_input_validation_optional_omit(): vector_json = r""" { "name": "input_validation_optional_omit", @@ -1098,7 +1151,7 @@ async def test_vector_30_loadconformance_load_input_validation_optional_omit(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_31_loadconformance_load_input_validation_required(): +async def test_vector_32_loadconformance_load_input_validation_required(): vector_json = r""" { "name": "input_validation_required", @@ -1129,7 +1182,7 @@ async def test_vector_31_loadconformance_load_input_validation_required(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_32_loadconformance_load_instructions_from_body(): +async def test_vector_33_loadconformance_load_instructions_from_body(): vector_json = r""" { "name": "instructions_from_body", @@ -1152,7 +1205,7 @@ async def test_vector_32_loadconformance_load_instructions_from_body(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_33_loadconformance_load_invalid_frontmatter_error(): +async def test_vector_34_loadconformance_load_invalid_frontmatter_error(): vector_json = r""" { "name": "invalid_frontmatter_error", @@ -1171,7 +1224,7 @@ async def test_vector_33_loadconformance_load_invalid_frontmatter_error(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_34_loadconformance_load_kind_always_prompt(): +async def test_vector_35_loadconformance_load_kind_always_prompt(): vector_json = r""" { "name": "kind_always_prompt", @@ -1190,7 +1243,7 @@ async def test_vector_34_loadconformance_load_kind_always_prompt(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_35_loadconformance_load_minimal_load(): +async def test_vector_36_loadconformance_load_minimal_load(): vector_json = r""" { "name": "minimal_load", @@ -1217,7 +1270,7 @@ async def test_vector_35_loadconformance_load_minimal_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_36_loadconformance_load_missing_file_error(): +async def test_vector_37_loadconformance_load_missing_file_error(): vector_json = r""" { "name": "missing_file_error", @@ -1236,7 +1289,7 @@ async def test_vector_36_loadconformance_load_missing_file_error(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_37_loadconformance_load_model_shorthand(): +async def test_vector_38_loadconformance_load_model_shorthand(): vector_json = r""" { "name": "model_shorthand", @@ -1262,7 +1315,7 @@ async def test_vector_37_loadconformance_load_model_shorthand(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_38_loadconformance_load_structured_outputs_load(): +async def test_vector_39_loadconformance_load_structured_outputs_load(): vector_json = r""" { "name": "structured_outputs_load", @@ -1328,7 +1381,7 @@ async def test_vector_38_loadconformance_load_structured_outputs_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_39_loadconformance_load_template_string_invalid(): +async def test_vector_40_loadconformance_load_template_string_invalid(): vector_json = r""" { "name": "template_string_invalid", @@ -1353,7 +1406,7 @@ async def test_vector_39_loadconformance_load_template_string_invalid(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_40_loadconformance_load_tools_custom_load(): +async def test_vector_41_loadconformance_load_tools_custom_load(): vector_json = r""" { "name": "tools_custom_load", @@ -1390,7 +1443,7 @@ async def test_vector_40_loadconformance_load_tools_custom_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_41_loadconformance_load_tools_function_load(): +async def test_vector_42_loadconformance_load_tools_function_load(): vector_json = r""" { "name": "tools_function_load", @@ -1456,7 +1509,7 @@ async def test_vector_41_loadconformance_load_tools_function_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_42_loadconformance_load_tools_mcp_load(): +async def test_vector_43_loadconformance_load_tools_mcp_load(): vector_json = r""" { "name": "tools_mcp_load", @@ -1495,7 +1548,7 @@ async def test_vector_42_loadconformance_load_tools_mcp_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_43_loadconformance_load_tools_openapi_load(): +async def test_vector_44_loadconformance_load_tools_openapi_load(): vector_json = r""" { "name": "tools_openapi_load", @@ -1534,7 +1587,7 @@ async def test_vector_43_loadconformance_load_tools_openapi_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_44_turnconformance_replay_max_iterations(): +async def test_vector_45_turnconformance_replay_max_iterations(): vector_json = r""" { "name": "max_iterations", @@ -1569,7 +1622,7 @@ async def test_vector_44_turnconformance_replay_max_iterations(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_45_turnconformance_replay_no_tool(): +async def test_vector_46_turnconformance_replay_no_tool(): vector_json = r""" { "name": "no_tool", @@ -1600,7 +1653,7 @@ async def test_vector_45_turnconformance_replay_no_tool(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_46_turnconformance_replay_permission_denied(): +async def test_vector_47_turnconformance_replay_permission_denied(): vector_json = r""" { "name": "permission_denied", @@ -1634,7 +1687,7 @@ async def test_vector_46_turnconformance_replay_permission_denied(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_47_turnconformance_replay_tool_failure(): +async def test_vector_48_turnconformance_replay_tool_failure(): vector_json = r""" { "name": "tool_failure", @@ -1670,7 +1723,7 @@ async def test_vector_47_turnconformance_replay_tool_failure(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_48_turnconformance_replay_tool_success(): +async def test_vector_49_turnconformance_replay_tool_success(): vector_json = r""" { "name": "tool_success", @@ -1706,7 +1759,7 @@ async def test_vector_48_turnconformance_replay_tool_success(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_49_turnconformance_run_assistant_tool_calls_metadata(): +async def test_vector_50_turnconformance_run_assistant_tool_calls_metadata(): vector_json = r""" { "name": "assistant_tool_calls_metadata", @@ -1858,7 +1911,7 @@ async def test_vector_49_turnconformance_run_assistant_tool_calls_metadata(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_50_turnconformance_run_async_tool_function(): +async def test_vector_51_turnconformance_run_async_tool_function(): vector_json = r""" { "name": "async_tool_function", @@ -1968,7 +2021,7 @@ async def test_vector_50_turnconformance_run_async_tool_function(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_51_turnconformance_run_bindings_injected(): +async def test_vector_52_turnconformance_run_bindings_injected(): vector_json = r""" { "name": "bindings_injected", @@ -2098,7 +2151,7 @@ async def test_vector_51_turnconformance_run_bindings_injected(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_52_turnconformance_run_cancellation_before_llm(): +async def test_vector_53_turnconformance_run_cancellation_before_llm(): vector_json = r""" { "name": "cancellation_before_llm", @@ -2159,7 +2212,7 @@ async def test_vector_52_turnconformance_run_cancellation_before_llm(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_53_turnconformance_run_cancellation_between_iterations(): +async def test_vector_54_turnconformance_run_cancellation_between_iterations(): vector_json = r""" { "name": "cancellation_between_iterations", @@ -2289,7 +2342,7 @@ async def test_vector_53_turnconformance_run_cancellation_between_iterations(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_54_turnconformance_run_cancellation_between_tools(): +async def test_vector_55_turnconformance_run_cancellation_between_tools(): vector_json = r""" { "name": "cancellation_between_tools", @@ -2428,7 +2481,7 @@ async def test_vector_54_turnconformance_run_cancellation_between_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_55_turnconformance_run_context_no_trim_when_fits(): +async def test_vector_56_turnconformance_run_context_no_trim_when_fits(): vector_json = r""" { "name": "context_no_trim_when_fits", @@ -2499,7 +2552,7 @@ async def test_vector_55_turnconformance_run_context_no_trim_when_fits(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_56_turnconformance_run_context_preserves_system_messages(): +async def test_vector_57_turnconformance_run_context_preserves_system_messages(): vector_json = r""" { "name": "context_preserves_system_messages", @@ -2606,7 +2659,7 @@ async def test_vector_56_turnconformance_run_context_preserves_system_messages() await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_57_turnconformance_run_context_trim_basic(): +async def test_vector_58_turnconformance_run_context_trim_basic(): vector_json = r""" { "name": "context_trim_basic", @@ -2764,7 +2817,7 @@ async def test_vector_57_turnconformance_run_context_trim_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_58_turnconformance_run_empty_tool_result(): +async def test_vector_59_turnconformance_run_empty_tool_result(): vector_json = r""" { "name": "empty_tool_result", @@ -2879,7 +2932,7 @@ async def test_vector_58_turnconformance_run_empty_tool_result(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_59_turnconformance_run_events_basic_tool_loop(): +async def test_vector_60_turnconformance_run_events_basic_tool_loop(): vector_json = r""" { "name": "events_basic_tool_loop", @@ -3025,7 +3078,7 @@ async def test_vector_59_turnconformance_run_events_basic_tool_loop(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_60_turnconformance_run_events_error_logged(): +async def test_vector_61_turnconformance_run_events_error_logged(): vector_json = r""" { "name": "events_error_logged", @@ -3151,7 +3204,7 @@ async def test_vector_60_turnconformance_run_events_error_logged(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_61_turnconformance_run_events_no_tools(): +async def test_vector_62_turnconformance_run_events_no_tools(): vector_json = r""" { "name": "events_no_tools", @@ -3229,7 +3282,7 @@ async def test_vector_61_turnconformance_run_events_no_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_62_turnconformance_run_guardrail_all_pass(): +async def test_vector_63_turnconformance_run_guardrail_all_pass(): vector_json = r""" { "name": "guardrail_all_pass", @@ -3352,7 +3405,7 @@ async def test_vector_62_turnconformance_run_guardrail_all_pass(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_63_turnconformance_run_guardrail_input_deny(): +async def test_vector_64_turnconformance_run_guardrail_input_deny(): vector_json = r""" { "name": "guardrail_input_deny", @@ -3408,7 +3461,7 @@ async def test_vector_63_turnconformance_run_guardrail_input_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_64_turnconformance_run_guardrail_output_deny(): +async def test_vector_65_turnconformance_run_guardrail_output_deny(): vector_json = r""" { "name": "guardrail_output_deny", @@ -3486,7 +3539,7 @@ async def test_vector_64_turnconformance_run_guardrail_output_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_65_turnconformance_run_guardrail_tool_deny(): +async def test_vector_66_turnconformance_run_guardrail_tool_deny(): vector_json = r""" { "name": "guardrail_tool_deny", @@ -3640,7 +3693,7 @@ async def test_vector_65_turnconformance_run_guardrail_tool_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_66_turnconformance_run_max_iterations_exceeded(): +async def test_vector_67_turnconformance_run_max_iterations_exceeded(): vector_json = r""" { "name": "max_iterations_exceeded", @@ -4150,7 +4203,7 @@ async def test_vector_66_turnconformance_run_max_iterations_exceeded(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_67_turnconformance_run_multi_turn_tool_calls(): +async def test_vector_68_turnconformance_run_multi_turn_tool_calls(): vector_json = r""" { "name": "multi_turn_tool_calls", @@ -4327,7 +4380,7 @@ async def test_vector_67_turnconformance_run_multi_turn_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_68_turnconformance_run_multiple_tool_calls_single_turn(): +async def test_vector_69_turnconformance_run_multiple_tool_calls_single_turn(): vector_json = r""" { "name": "multiple_tool_calls_single_turn", @@ -4508,7 +4561,7 @@ async def test_vector_68_turnconformance_run_multiple_tool_calls_single_turn(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_69_turnconformance_run_no_tool_calls(): +async def test_vector_70_turnconformance_run_no_tool_calls(): vector_json = r""" { "name": "no_tool_calls", @@ -4576,7 +4629,7 @@ async def test_vector_69_turnconformance_run_no_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_70_turnconformance_run_parallel_tools_basic(): +async def test_vector_71_turnconformance_run_parallel_tools_basic(): vector_json = r""" { "name": "parallel_tools_basic", @@ -4750,7 +4803,7 @@ async def test_vector_70_turnconformance_run_parallel_tools_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_71_turnconformance_run_parallel_tools_with_guardrail_deny(): +async def test_vector_72_turnconformance_run_parallel_tools_with_guardrail_deny(): vector_json = r""" { "name": "parallel_tools_with_guardrail_deny", @@ -4942,7 +4995,7 @@ async def test_vector_71_turnconformance_run_parallel_tools_with_guardrail_deny( await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_72_turnconformance_run_single_tool_call(): +async def test_vector_73_turnconformance_run_single_tool_call(): vector_json = r""" { "name": "single_tool_call", @@ -5089,7 +5142,7 @@ async def test_vector_72_turnconformance_run_single_tool_call(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_73_turnconformance_run_steering_inject_message(): +async def test_vector_74_turnconformance_run_steering_inject_message(): vector_json = r""" { "name": "steering_inject_message", @@ -5297,7 +5350,7 @@ async def test_vector_73_turnconformance_run_steering_inject_message(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_74_turnconformance_run_steering_multiple_messages(): +async def test_vector_75_turnconformance_run_steering_multiple_messages(): vector_json = r""" { "name": "steering_multiple_messages", @@ -5467,7 +5520,7 @@ async def test_vector_74_turnconformance_run_steering_multiple_messages(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_75_turnconformance_run_tool_not_registered_error(): +async def test_vector_76_turnconformance_run_tool_not_registered_error(): vector_json = r""" { "name": "tool_not_registered_error", @@ -5551,7 +5604,7 @@ async def test_vector_75_turnconformance_run_tool_not_registered_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_76_turnconformance_run_tool_result_message_format(): +async def test_vector_77_turnconformance_run_tool_result_message_format(): vector_json = r""" { "name": "tool_result_message_format", @@ -5672,7 +5725,7 @@ async def test_vector_76_turnconformance_run_tool_result_message_format(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_77_turnconformance_runturn_cancel_before_context(): +async def test_vector_78_turnconformance_runturn_cancel_before_context(): vector_json = r""" { "name": "cancel_before_context", @@ -5704,7 +5757,7 @@ async def test_vector_77_turnconformance_runturn_cancel_before_context(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_78_turnconformance_runturn_delegated_provider_state(): +async def test_vector_79_turnconformance_runturn_delegated_provider_state(): vector_json = r""" { "name": "delegated_provider_state", @@ -5771,7 +5824,7 @@ async def test_vector_78_turnconformance_runturn_delegated_provider_state(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_79_turnconformance_runturn_final_output(): +async def test_vector_80_turnconformance_runturn_final_output(): vector_json = r""" { "name": "final_output", @@ -5816,7 +5869,7 @@ async def test_vector_79_turnconformance_runturn_final_output(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_80_turnconformance_runturn_ordered_tool_round(): +async def test_vector_81_turnconformance_runturn_ordered_tool_round(): vector_json = r""" { "name": "ordered_tool_round", @@ -5907,7 +5960,7 @@ async def test_vector_80_turnconformance_runturn_ordered_tool_round(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_81_turnconformance_runturn_permission_denial_is_model_visible(): +async def test_vector_82_turnconformance_runturn_permission_denial_is_model_visible(): vector_json = r""" { "name": "permission_denial_is_model_visible", @@ -5958,7 +6011,7 @@ async def test_vector_81_turnconformance_runturn_permission_denial_is_model_visi await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_82_wireconformance_torequest_anthropic_image_format(): +async def test_vector_83_wireconformance_torequest_anthropic_image_format(): vector_json = r""" { "name": "anthropic_image_format", @@ -6028,7 +6081,7 @@ async def test_vector_82_wireconformance_torequest_anthropic_image_format(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_83_wireconformance_torequest_anthropic_max_tokens_required(): +async def test_vector_84_wireconformance_torequest_anthropic_max_tokens_required(): vector_json = r""" { "name": "anthropic_max_tokens_required", @@ -6083,7 +6136,7 @@ async def test_vector_83_wireconformance_torequest_anthropic_max_tokens_required await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_84_wireconformance_torequest_anthropic_options(): +async def test_vector_85_wireconformance_torequest_anthropic_options(): vector_json = r""" { "name": "anthropic_options", @@ -6152,7 +6205,7 @@ async def test_vector_84_wireconformance_torequest_anthropic_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_85_wireconformance_torequest_anthropic_system_separate(): +async def test_vector_86_wireconformance_torequest_anthropic_system_separate(): vector_json = r""" { "name": "anthropic_system_separate", @@ -6219,7 +6272,7 @@ async def test_vector_85_wireconformance_torequest_anthropic_system_separate(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_86_wireconformance_torequest_anthropic_tool_wire(): +async def test_vector_87_wireconformance_torequest_anthropic_tool_wire(): vector_json = r""" { "name": "anthropic_tool_wire", @@ -6304,7 +6357,7 @@ async def test_vector_86_wireconformance_torequest_anthropic_tool_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_87_wireconformance_torequest_chat_audio_mp3(): +async def test_vector_88_wireconformance_torequest_chat_audio_mp3(): vector_json = r""" { "name": "chat_audio_mp3", @@ -6362,7 +6415,7 @@ async def test_vector_87_wireconformance_torequest_chat_audio_mp3(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_88_wireconformance_torequest_chat_audio_part(): +async def test_vector_89_wireconformance_torequest_chat_audio_part(): vector_json = r""" { "name": "chat_audio_part", @@ -6420,7 +6473,7 @@ async def test_vector_88_wireconformance_torequest_chat_audio_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_89_wireconformance_torequest_chat_image_base64(): +async def test_vector_90_wireconformance_torequest_chat_image_base64(): vector_json = r""" { "name": "chat_image_base64", @@ -6484,7 +6537,7 @@ async def test_vector_89_wireconformance_torequest_chat_image_base64(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_90_wireconformance_torequest_chat_image_part(): +async def test_vector_91_wireconformance_torequest_chat_image_part(): vector_json = r""" { "name": "chat_image_part", @@ -6548,7 +6601,7 @@ async def test_vector_90_wireconformance_torequest_chat_image_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_91_wireconformance_torequest_chat_multipart_content(): +async def test_vector_92_wireconformance_torequest_chat_multipart_content(): vector_json = r""" { "name": "chat_multipart_content", @@ -6612,7 +6665,7 @@ async def test_vector_91_wireconformance_torequest_chat_multipart_content(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_92_wireconformance_torequest_chat_simple(): +async def test_vector_93_wireconformance_torequest_chat_simple(): vector_json = r""" { "name": "chat_simple", @@ -6674,7 +6727,7 @@ async def test_vector_92_wireconformance_torequest_chat_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_93_wireconformance_torequest_chat_single_text_optimized(): +async def test_vector_94_wireconformance_torequest_chat_single_text_optimized(): vector_json = r""" { "name": "chat_single_text_optimized", @@ -6723,7 +6776,7 @@ async def test_vector_93_wireconformance_torequest_chat_single_text_optimized(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_94_wireconformance_torequest_chat_with_options(): +async def test_vector_95_wireconformance_torequest_chat_with_options(): vector_json = r""" { "name": "chat_with_options", @@ -6791,7 +6844,7 @@ async def test_vector_94_wireconformance_torequest_chat_with_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_95_wireconformance_torequest_embedding_wire(): +async def test_vector_96_wireconformance_torequest_embedding_wire(): vector_json = r""" { "name": "embedding_wire", @@ -6835,7 +6888,7 @@ async def test_vector_95_wireconformance_torequest_embedding_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_96_wireconformance_torequest_image_wire(): +async def test_vector_97_wireconformance_torequest_image_wire(): vector_json = r""" { "name": "image_wire", @@ -6879,7 +6932,7 @@ async def test_vector_96_wireconformance_torequest_image_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_97_wireconformance_torequest_kind_to_json_type_mapping(): +async def test_vector_98_wireconformance_torequest_kind_to_json_type_mapping(): vector_json = r""" { "name": "kind_to_json_type_mapping", @@ -6998,7 +7051,7 @@ async def test_vector_97_wireconformance_torequest_kind_to_json_type_mapping(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_98_wireconformance_torequest_options_additional_properties(): +async def test_vector_99_wireconformance_torequest_options_additional_properties(): vector_json = r""" { "name": "options_additional_properties", @@ -7052,7 +7105,7 @@ async def test_vector_98_wireconformance_torequest_options_additional_properties await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_99_wireconformance_torequest_options_max_completion_tokens(): +async def test_vector_100_wireconformance_torequest_options_max_completion_tokens(): vector_json = r""" { "name": "options_max_completion_tokens", @@ -7104,7 +7157,7 @@ async def test_vector_99_wireconformance_torequest_options_max_completion_tokens await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_100_wireconformance_torequest_options_stop_sequences(): +async def test_vector_101_wireconformance_torequest_options_stop_sequences(): vector_json = r""" { "name": "options_stop_sequences", @@ -7162,7 +7215,7 @@ async def test_vector_100_wireconformance_torequest_options_stop_sequences(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_101_wireconformance_torequest_responses_simple(): +async def test_vector_102_wireconformance_torequest_responses_simple(): vector_json = r""" { "name": "responses_simple", @@ -7221,7 +7274,7 @@ async def test_vector_101_wireconformance_torequest_responses_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_102_wireconformance_torequest_responses_structured_output(): +async def test_vector_103_wireconformance_torequest_responses_structured_output(): vector_json = r""" { "name": "responses_structured_output", @@ -7308,7 +7361,7 @@ async def test_vector_102_wireconformance_torequest_responses_structured_output( await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_103_wireconformance_torequest_responses_with_tools(): +async def test_vector_104_wireconformance_torequest_responses_with_tools(): vector_json = r""" { "name": "responses_with_tools", @@ -7388,7 +7441,7 @@ async def test_vector_103_wireconformance_torequest_responses_with_tools(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_104_wireconformance_torequest_structured_output(): +async def test_vector_105_wireconformance_torequest_structured_output(): vector_json = r""" { "name": "structured_output", @@ -7471,7 +7524,7 @@ async def test_vector_104_wireconformance_torequest_structured_output(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_105_wireconformance_torequest_structured_output_nested_optional(): +async def test_vector_106_wireconformance_torequest_structured_output_nested_optional(): vector_json = r""" { "name": "structured_output_nested_optional", @@ -7573,7 +7626,7 @@ async def test_vector_105_wireconformance_torequest_structured_output_nested_opt await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_106_wireconformance_torequest_tools_bindings_stripped(): +async def test_vector_107_wireconformance_torequest_tools_bindings_stripped(): vector_json = r""" { "name": "tools_bindings_stripped", @@ -7665,7 +7718,7 @@ async def test_vector_106_wireconformance_torequest_tools_bindings_stripped(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_107_wireconformance_torequest_tools_function_wire(): +async def test_vector_108_wireconformance_torequest_tools_function_wire(): vector_json = r""" { "name": "tools_function_wire", @@ -7747,7 +7800,7 @@ async def test_vector_107_wireconformance_torequest_tools_function_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_108_wireconformance_torequest_tools_null_when_empty(): +async def test_vector_109_wireconformance_torequest_tools_null_when_empty(): vector_json = r""" { "name": "tools_null_when_empty", @@ -7796,7 +7849,7 @@ async def test_vector_108_wireconformance_torequest_tools_null_when_empty(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_109_wireconformance_torequest_tools_strict_mode(): +async def test_vector_110_wireconformance_torequest_tools_strict_mode(): vector_json = r""" { "name": "tools_strict_mode", diff --git a/runtime/python/prompty/tests/model/vector_adapters.py b/runtime/python/prompty/tests/model/vector_adapters.py index 8f9d3a448..a96d84afb 100644 --- a/runtime/python/prompty/tests/model/vector_adapters.py +++ b/runtime/python/prompty/tests/model/vector_adapters.py @@ -983,28 +983,27 @@ def _live_provider_impls(provider: str) -> tuple[Any, Any]: raise ValueError(f"live-chat adapter: unknown provider {provider!r}") -def _build_live_agent(resolved_input: dict) -> Any: - """Construct a chat ``Agent`` from a resolved live-chat vector input.""" +def _drill_build_agent(resolved_input: dict, *, endpoint: str | None, api_key: str | None) -> Any: + """Construct a chat ``Agent`` bound to a specific transport ``endpoint``. + + The drill binds the transport seam via base-URL redirect, so ``endpoint`` and + ``api_key`` are supplied by the caller (a local cassette-replay server in + replay mode, the real base URL + key in live mode) rather than read from the + vector input. When ``endpoint`` is falsy the connection omits it so the SDK + falls back to the provider's default base URL (live mode without an override). + """ from prompty.model import Agent provider = (resolved_input.get("provider") or "openai").lower() - api_key = resolved_input.get("apiKey") - endpoint = resolved_input.get("endpoint") - - if provider in ("foundry", "azure") and not api_key: - # Entra ID path: no key, DefaultAzureCredential drives auth. - connection: dict[str, Any] = {"kind": "foundry"} - if endpoint: - connection["endpoint"] = endpoint - else: - connection = {"kind": "key"} - if api_key: - connection["apiKey"] = api_key - if endpoint: - connection["endpoint"] = endpoint + + connection: dict[str, Any] = {"kind": "key"} + if api_key: + connection["apiKey"] = api_key + if endpoint: + connection["endpoint"] = endpoint data: dict[str, Any] = { - "name": "live-chat-vector", + "name": "drill-chat-vector", "model": { "id": resolved_input.get("model") or "gpt-4o-mini", "provider": provider, @@ -1056,25 +1055,63 @@ def _extract_chat_structure(observed: Any) -> tuple[Any, Any, Any]: return role, content, finish -def _live_chat_invoke(resolved_input: dict, context: dict) -> Any: - """Call a real provider and return its raw chat response. +# --------------------------------------------------------------------------- +# LiveChatConformance.complete -- the service-drill adapter +# --------------------------------------------------------------------------- +# +# A "drill" is a conformance vector that crosses the REAL Executor transport +# seam. Unlike the pure conformance vectors that stop at the SDK edge, the drill +# binds the seam via base-URL redirect -- to a local ``CassetteReplayServer`` +# (replay mode) or the real base URL (live mode) -- and asserts three planes: +# +# 1. transport -- the seam was reached and returned bytes (executor got a +# response, and the replay server captured an inbound request). +# 2. wire -- the outbound provider request body matches the cassette's +# recorded request body (canonical equality; throw on mismatch). +# 3. semantic -- normalized executor output == the vector `expected`. This +# plane is asserted by the RUNNER via the registered ``normalize``, so the +# drill's semantic plane IS a conformance vector over the executor. +# +# In live mode the adapter additionally asserts live-semantic == replay-semantic +# (parity). ``requires`` is deliberately OMITTED from the drill vector -- the +# ``requires`` guard is all-or-nothing and would skip the WHOLE vector when creds +# are absent, defeating the replay plane. Instead the adapter self-gates the live +# plane on credential presence and always runs replay. +# +# Cassette bytes and the SDK transport binding are consumer-owned "roster" -- by +# design NOT modeled in TypeSpec/Typra. - Only reached when the capability guard has already confirmed the required - credentials/capabilities are present, so this makes a genuine network call. - """ +_DRILL_REPLAY_API_KEY = "sk-drill-replay-placeholder" + + +def _drill_canonical(value: Any) -> str: + """Order-insensitive canonical JSON for wire/parity equality checks.""" + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) + + +def _drill_build_messages(resolved_input: dict) -> list: from prompty.core.types import Message, TextPart - messages_spec = resolved_input.get("messages") or [] - messages = [ + return [ Message(role=spec.get("role", "user"), parts=[TextPart(value=spec.get("content", ""))]) - for spec in messages_spec + for spec in (resolved_input.get("messages") or []) ] - agent = _build_live_agent(resolved_input) + + +def _drill_execute(resolved_input: dict, *, endpoint: str | None, api_key: str | None) -> Any: + """Drive the real executor with its transport bound to ``endpoint``. + + Returns the raw provider response. Shared by the replay path (``endpoint`` = + a ``CassetteReplayServer`` base URL) and the live path (``endpoint`` = the + real base URL or ``None`` for the SDK default). + """ + agent = _drill_build_agent(resolved_input, endpoint=endpoint, api_key=api_key) executor, _processor = _live_provider_impls(resolved_input.get("provider") or "openai") + messages = _drill_build_messages(resolved_input) return executor.execute(agent, messages) -def _live_chat_normalize(observed: Any, context: dict) -> dict: +def _drill_normalize_response(observed: Any) -> dict: """Project a raw chat response onto the canonical structural shape.""" role, content, finish = _extract_chat_structure(observed) return { @@ -1084,6 +1121,82 @@ def _live_chat_normalize(observed: Any, context: dict) -> dict: } +def _drill_live_enabled(resolved_input: dict) -> bool: + """Live plane is enabled iff a base URL AND an API key resolved from env. + + ``$env`` refs resolve to ``""`` when unset, so this is false in replay-only + (no-creds) runs and true only when the vector's endpoint+apiKey env vars are + both populated. + """ + return bool(resolved_input.get("endpoint") and resolved_input.get("apiKey")) + + +def _drill_invoke(resolved_input: dict, context: dict) -> Any: + """Run the drill: bind transport, replay the cassette, assert 3 planes. + + Returns the raw replay response; the runner then applies ``normalize`` and + compares against the vector ``expected`` (the semantic plane). + """ + from _drill_replay import CassetteReplayServer + + resolve = context.get("resolveInput") + vector = context.get("vector") or {} + exchange_raw = vector.get("exchange") or {} + # The runner resolves refs only on `input`; the adapter owns `exchange` + # resolution (nested `$env` transport ref + `$json` cassette ref). + exchange = resolve(exchange_raw) if callable(resolve) else exchange_raw + + cassette = exchange.get("cassette") or {} + recorded_request = (cassette.get("request") or {}).get("body") + recorded_response = (cassette.get("response") or {}).get("body") + recorded_status = int((cassette.get("response") or {}).get("status", 200)) + if recorded_response is None: + raise AssertionError("drill cassette missing response.body") + + # --- Replay: bind the transport to the local cassette server ------------- + with CassetteReplayServer(recorded_response, recorded_status) as server: + raw = _drill_execute(resolved_input, endpoint=server.base_url, api_key=_DRILL_REPLAY_API_KEY) + + # Plane 1: transport -- the seam was reached and a request was captured. + captured = server.last_request + if captured is None: + raise AssertionError("drill transport plane: no request reached the replay server") + if raw is None: + raise AssertionError("drill transport plane: executor returned no response") + + # Plane 2: wire -- outbound request body matches the cassette request. + if recorded_request is not None: + observed_body = captured.get("body") + if _drill_canonical(observed_body) != _drill_canonical(recorded_request): + raise AssertionError( + "drill wire plane mismatch:\n" + f" observed={_drill_canonical(observed_body)}\n" + f" cassette={_drill_canonical(recorded_request)}" + ) + + # --- Live parity (self-gated): live-semantic must equal replay-semantic --- + if _drill_live_enabled(resolved_input): + live_raw = _drill_execute( + resolved_input, + endpoint=resolved_input.get("endpoint") or None, + api_key=resolved_input.get("apiKey"), + ) + if _drill_canonical(_drill_normalize_response(live_raw)) != _drill_canonical(_drill_normalize_response(raw)): + raise AssertionError( + "drill live/replay parity mismatch:\n" + f" live={_drill_normalize_response(live_raw)}\n" + f" replay={_drill_normalize_response(raw)}" + ) + + # Plane 3 (semantic) is asserted by the runner: normalize(raw) == expected. + return raw + + +def _drill_normalize(observed: Any, context: dict) -> dict: + """Registered semantic projection: raw response -> canonical structure.""" + return _drill_normalize_response(observed) + + # --------------------------------------------------------------------------- # Adapter registry # --------------------------------------------------------------------------- @@ -1096,7 +1209,7 @@ def _live_chat_normalize(observed: Any, context: dict) -> dict: "DiscoveryConformance.enrich": {"invoke": _discovery_enrich_invoke}, "DiscoveryConformance.mapModel": {"invoke": _discovery_map_invoke}, "TurnConformance.replay": {"invoke": _replay_invoke}, - "LiveChatConformance.complete": {"invoke": _live_chat_invoke, "normalize": _live_chat_normalize}, + "LiveChatConformance.complete": {"invoke": _drill_invoke, "normalize": _drill_normalize}, } # Contracts introduced/tightened by Typra 0.12.0 that the Python runtime does not diff --git a/runtime/rust/prompty/src/model/live_chat_conformance.rs b/runtime/rust/prompty/src/model/live_chat_conformance.rs new file mode 100644 index 000000000..15ccb14b0 --- /dev/null +++ b/runtime/rust/prompty/src/model/live_chat_conformance.rs @@ -0,0 +1,21 @@ +// +// Code generated by Typra emitter; DO NOT EDIT. + +#![allow( + unused_imports, + dead_code, + non_camel_case_types, + unused_variables, + unexpected_cfgs, + clippy::all +)] + +/// Conformance-only seam hosting the "service drill" vectors: a conformance vector that crosses the REAL Executor->Processor transport seam instead of stopping at the SDK edge. A drill runs against a live provider when credentials are present and REPLAYS a recorded cassette when they are not; the live<->replay parity is the gate. Like every other conformance seam this interface exists purely to carry +#[async_trait::async_trait] +pub trait LiveChatConformance: Send + Sync { + /// Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes + async fn complete( + &self, + input: &serde_json::Value, + ) -> Result>; +} diff --git a/runtime/rust/prompty/src/model/mod.rs b/runtime/rust/prompty/src/model/mod.rs index 84a3bea3b..5c8027190 100644 --- a/runtime/rust/prompty/src/model/mod.rs +++ b/runtime/rust/prompty/src/model/mod.rs @@ -46,6 +46,9 @@ pub use executor::*; pub mod host_tool_executor; pub use host_tool_executor::*; +pub mod live_chat_conformance; +pub use live_chat_conformance::*; + pub mod load_conformance; pub use load_conformance::*; diff --git a/runtime/rust/prompty/tests/model/vector_adapters.rs b/runtime/rust/prompty/tests/model/vector_adapters.rs index 2cf961295..edab7f1c4 100644 --- a/runtime/rust/prompty/tests/model/vector_adapters.rs +++ b/runtime/rust/prompty/tests/model/vector_adapters.rs @@ -201,7 +201,10 @@ pub fn adapters() -> HashMap<&'static str, Adapter> { } pub fn waivers() -> HashMap<&'static str, &'static str> { - HashMap::new() + HashMap::from([( + "LiveChatConformance.complete", + "drill: consumer-owned transport-seam probe (Python/TS only)", + )]) } pub fn doubles() -> Value { diff --git a/runtime/rust/prompty/tests/model/vector_conformance_test.rs b/runtime/rust/prompty/tests/model/vector_conformance_test.rs index 7578fa9b4..b1970acf1 100644 --- a/runtime/rust/prompty/tests/model/vector_conformance_test.rs +++ b/runtime/rust/prompty/tests/model/vector_conformance_test.rs @@ -173,658 +173,665 @@ async fn test_vector_15_discoveryconformance_mapmodel_openai_model_finetune_no_o } #[tokio::test] -async fn test_vector_16_loadconformance_load_basic_load() { +async fn test_vector_16_livechatconformance_complete_openai_chat_drill() { + let vector: Value = serde_json::from_str("{\"name\":\"openai_chat_drill\",\"stage\":\"drill\",\"provider\":\"openai\",\"input\":{\"provider\":\"openai\",\"model\":\"gpt-4o-mini\",\"apiKey\":{\"$env\":\"OPENAI_API_KEY\"},\"endpoint\":{\"$env\":\"OPENAI_BASE_URL\"},\"messages\":[{\"role\":\"user\",\"content\":\"Say hello in exactly one word.\"}],\"options\":{\"temperature\":0,\"maxOutputTokens\":16}},\"exchange\":{\"transport\":{\"baseUrl\":{\"$env\":\"OPENAI_BASE_URL\"}},\"cassette\":{\"$json\":\"cassettes/openai_chat_drill.json\"},\"planes\":[\"transport\",\"wire\",\"semantic\"]},\"expected\":{\"role\":\"assistant\",\"contentNonEmpty\":true,\"finishReasonInEnum\":true},\"operation\":\"complete\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LiveChatConformance", "complete", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_17_loadconformance_load_basic_load() { let vector: Value = serde_json::from_str("{\"name\":\"basic_load\",\"description\":\"Load basic.prompty with env var resolution and verify all PromptAgent fields\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"basic-prompt\",\"description\":\"A basic prompt for testing\",\"metadata\":{\"authors\":[\"testauthor\"]},\"model\":{\"id\":\"gpt-4\",\"provider\":\"openai\",\"apiType\":\"chat\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"},\"options\":{\"temperature\":0.7,\"maxOutputTokens\":1000}},\"inputs\":[{\"name\":\"firstName\",\"kind\":\"string\",\"default\":\"Jane\"},{\"name\":\"lastName\",\"kind\":\"string\",\"default\":\"Doe\"},{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What is the meaning of life?\"}],\"template\":{\"format\":{\"kind\":\"jinja2\"},\"parser\":{\"kind\":\"prompty\"}},\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_17_loadconformance_load_connection_types_load() { +async fn test_vector_18_loadconformance_load_connection_types_load() { let vector: Value = serde_json::from_str("{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_18_loadconformance_load_embedding_load() { +async fn test_vector_19_loadconformance_load_embedding_load() { let vector: Value = serde_json::from_str("{\"name\":\"embedding_load\",\"description\":\"Load embedding.prompty and verify apiType=embedding with correct model id\",\"stage\":\"load\",\"input\":{\"fixture\":\"embedding.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"embedding\",\"model\":{\"id\":\"text-embedding-3-small\",\"provider\":\"openai\",\"apiType\":\"embedding\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"}},\"inputs\":[{\"name\":\"text\",\"kind\":\"string\",\"default\":\"Hello world\"}],\"instructions\":\"{{text}}\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_19_loadconformance_load_empty_frontmatter_body_only() { +async fn test_vector_20_loadconformance_load_empty_frontmatter_body_only() { let vector: Value = serde_json::from_str("{\"name\":\"empty_frontmatter_body_only\",\"description\":\"Empty frontmatter with body \u{2014} instructions extracted, defaults applied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"empty-fm\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"empty-fm\",\"model\":null,\"inputs\":null,\"tools\":null},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_20_loadconformance_load_env_default() { +async fn test_vector_21_loadconformance_load_env_default() { let vector: Value = serde_json::from_str("{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_21_loadconformance_load_env_missing_error() { +async fn test_vector_22_loadconformance_load_env_missing_error() { let vector: Value = serde_json::from_str("{\"name\":\"env_missing_error\",\"description\":\"${env:NONEXISTENT} with no default and var not set raises an error\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-error-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:NONEXISTENT}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"env_var_not_set\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_22_loadconformance_load_env_resolution() { +async fn test_vector_23_loadconformance_load_env_resolution() { let vector: Value = serde_json::from_str("{\"name\":\"env_resolution\",\"description\":\"${env:MY_VAR} resolves to the environment variable value when set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MY_VAR}\"}}},\"env\":{\"MY_VAR\":\"hello\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"hello\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_23_loadconformance_load_file_reference_in_tree_allowed() { +async fn test_vector_24_loadconformance_load_file_reference_in_tree_allowed() { let vector: Value = serde_json::from_str("{\"name\":\"file_reference_in_tree_allowed\",\"description\":\"A ${file:sub/data.json} reference that stays within the prompt file's directory tree resolves to the parsed content (containment control alongside the traversal-reject vector)\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-in-tree-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:sub/data.json}\"}},\"files\":{\"sub/data.json\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\",\"apiKey\":\"ok\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_24_loadconformance_load_file_reference_traversal_rejected() { +async fn test_vector_25_loadconformance_load_file_reference_traversal_rejected() { let vector: Value = serde_json::from_str("{\"name\":\"file_reference_traversal_rejected\",\"description\":\"A ${file:../secret.json} reference that escapes the prompt file's directory via `..` is rejected even though the target file exists, preventing arbitrary file disclosure (GHSA-7pfc-5v9r-j299). Uses forward-slash separators so the vector holds on every OS.\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-traversal-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:../secret.json}\"}},\"files\":{\"../secret.json\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"leaked\"}}},\"expectedError\":{\"kind\":\"file_reference\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_25_loadconformance_load_file_resolution() { +async fn test_vector_26_loadconformance_load_file_resolution() { let vector: Value = serde_json::from_str("{\"name\":\"file_resolution\",\"description\":\"${file:shared_connection.json} resolves to the parsed JSON content of the referenced file\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"file-ref-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:shared_connection.json}\"}},\"env\":{\"OPENAI_API_KEY\":\"shared-key\"},\"files\":{\"shared_connection.json\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\",\"apiKey\":\"shared-key\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_26_loadconformance_load_image_apitype_load() { +async fn test_vector_27_loadconformance_load_image_apitype_load() { let vector: Value = serde_json::from_str("{\"name\":\"image_apitype_load\",\"description\":\"An image-generation prompt loads with apiType image and passthrough additionalProperties model options\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}}},\"expected\":{\"kind\":\"prompt\",\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_27_loadconformance_load_input_scalar_shorthand() { +async fn test_vector_28_loadconformance_load_input_scalar_shorthand() { let vector: Value = serde_json::from_str("{\"name\":\"input_scalar_shorthand\",\"description\":\"Scalar input values create typed Property with default set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"scalar-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]}},\"expected\":{\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_28_loadconformance_load_input_validation_default_fill() { +async fn test_vector_29_loadconformance_load_input_validation_default_fill() { let vector: Value = serde_json::from_str("{\"name\":\"input_validation_default_fill\",\"description\":\"An input with a default value is filled in when no value is provided\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"default-fill-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"name\",\"kind\":\"string\",\"default\":\"world\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{\"name\":\"world\"}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_29_loadconformance_load_input_validation_example_not_used() { +async fn test_vector_30_loadconformance_load_input_validation_example_not_used() { let vector: Value = serde_json::from_str("{\"name\":\"input_validation_example_not_used\",\"description\":\"An input with example='sample' but no default must NOT use the example at runtime\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"example-not-used-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"required\":false,\"example\":\"sample\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_30_loadconformance_load_input_validation_optional_omit() { +async fn test_vector_31_loadconformance_load_input_validation_optional_omit() { let vector: Value = serde_json::from_str("{\"name\":\"input_validation_optional_omit\",\"description\":\"A non-required input with no default and no value is omitted (not an error)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"optional-omit-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"nickname\",\"kind\":\"string\",\"required\":false}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_31_loadconformance_load_input_validation_required() { +async fn test_vector_32_loadconformance_load_input_validation_required() { let vector: Value = serde_json::from_str("{\"name\":\"input_validation_required\",\"description\":\"A required input with no default and no value provided raises ValueError\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"required-input-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},\"inputs\":{}},\"expectedError\":{\"kind\":\"missing_required_input\",\"field\":\"city\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_32_loadconformance_load_instructions_from_body() { +async fn test_vector_33_loadconformance_load_instructions_from_body() { let vector: Value = serde_json::from_str("{\"name\":\"instructions_from_body\",\"description\":\"The markdown body after the closing --- becomes the instructions field\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_33_loadconformance_load_invalid_frontmatter_error() { +async fn test_vector_34_loadconformance_load_invalid_frontmatter_error() { let vector: Value = serde_json::from_str("{\"name\":\"invalid_frontmatter_error\",\"description\":\"Malformed YAML in frontmatter raises a parse error\",\"stage\":\"load\",\"input\":{\"frontmatter_raw\":\"---\\nname: [invalid\\n---\\nHello\"},\"expectedError\":{\"kind\":\"invalid_frontmatter\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_34_loadconformance_load_kind_always_prompt() { +async fn test_vector_35_loadconformance_load_kind_always_prompt() { let vector: Value = serde_json::from_str("{\"name\":\"kind_always_prompt\",\"description\":\"The loader always injects kind='prompt' \u{2014} .prompty files never specify kind themselves\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_35_loadconformance_load_minimal_load() { +async fn test_vector_36_loadconformance_load_minimal_load() { let vector: Value = serde_json::from_str("{\"name\":\"minimal_load\",\"description\":\"Load minimal.prompty with model shorthand and bare-minimum frontmatter\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"minimal\",\"model\":{\"id\":\"gpt-4\"},\"instructions\":\"system:\\nHello world.\",\"inputs\":null,\"outputs\":null,\"tools\":null},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_36_loadconformance_load_missing_file_error() { +async fn test_vector_37_loadconformance_load_missing_file_error() { let vector: Value = serde_json::from_str("{\"name\":\"missing_file_error\",\"description\":\"Loading a nonexistent .prompty file raises FileNotFoundError\",\"stage\":\"load\",\"input\":{\"fixture\":\"nonexistent.prompty\"},\"expectedError\":{\"kind\":\"file_not_found\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_37_loadconformance_load_model_shorthand() { +async fn test_vector_38_loadconformance_load_model_shorthand() { let vector: Value = serde_json::from_str("{\"name\":\"model_shorthand\",\"description\":\"String shorthand 'model: gpt-4o' expands to a Model object with id set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"test\",\"model\":\"gpt-4o\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"test\",\"model\":{\"id\":\"gpt-4o\"}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_38_loadconformance_load_structured_outputs_load() { +async fn test_vector_39_loadconformance_load_structured_outputs_load() { let vector: Value = serde_json::from_str("{\"name\":\"structured_outputs_load\",\"description\":\"Outputs load as a list of properties preserving kind, description, and required flags\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"structured-output\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]}},\"expected\":{\"kind\":\"prompt\",\"name\":\"structured-output\",\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_39_loadconformance_load_template_string_invalid() { +async fn test_vector_40_loadconformance_load_template_string_invalid() { let vector: Value = serde_json::from_str("{\"name\":\"template_string_invalid\",\"description\":\"Template as a bare string is not valid v2 \u{2014} must be an object with format/parser\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"shorthand-test\",\"model\":{\"id\":\"gpt-4\"},\"template\":\"jinja2\"}},\"expectedError\":{\"kind\":\"invalid_template\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_40_loadconformance_load_tools_custom_load() { +async fn test_vector_41_loadconformance_load_tools_custom_load() { let vector: Value = serde_json::from_str("{\"name\":\"tools_custom_load\",\"description\":\"Load a prompty with an unknown tool kind \u{2014} falls through to CustomTool\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"custom-tool-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://custom.example.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\"}]},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_41_loadconformance_load_tools_function_load() { +async fn test_vector_42_loadconformance_load_tools_function_load() { let vector: Value = serde_json::from_str("{\"name\":\"tools_function_load\",\"description\":\"Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings\",\"stage\":\"load\",\"input\":{\"fixture\":\"tools_function.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"function-tools\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather in a given location\",\"strict\":true,\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"description\":\"The city and state, e.g. San Francisco, CA\"},{\"name\":\"unit\",\"kind\":\"string\",\"description\":\"Temperature unit\",\"enumValues\":[\"celsius\",\"fahrenheit\"]}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"inputs\":[{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What's the weather in Paris?\"},{\"name\":\"preferred_unit\",\"kind\":\"string\",\"default\":\"celsius\"}],\"instructions\":\"system:\\nYou are a helpful assistant with access to tools.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_42_loadconformance_load_tools_mcp_load() { +async fn test_vector_43_loadconformance_load_tools_mcp_load() { let vector: Value = serde_json::from_str("{\"name\":\"tools_mcp_load\",\"description\":\"Load a prompty with an MCP tool \u{2014} kind, serverName, connection preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"mcp-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-mcp\"}}]}},\"expected\":{\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\"}]},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_43_loadconformance_load_tools_openapi_load() { +async fn test_vector_44_loadconformance_load_tools_openapi_load() { let vector: Value = serde_json::from_str("{\"name\":\"tools_openapi_load\",\"description\":\"Load a prompty with an OpenAPI tool \u{2014} specification path preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"openapi-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://api.weather.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\"}]},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_44_turnconformance_replay_max_iterations() { +async fn test_vector_45_turnconformance_replay_max_iterations() { let vector: Value = serde_json::from_str("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_45_turnconformance_replay_no_tool() { +async fn test_vector_46_turnconformance_replay_no_tool() { let vector: Value = serde_json::from_str("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_46_turnconformance_replay_permission_denied() { +async fn test_vector_47_turnconformance_replay_permission_denied() { let vector: Value = serde_json::from_str("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_47_turnconformance_replay_tool_failure() { +async fn test_vector_48_turnconformance_replay_tool_failure() { let vector: Value = serde_json::from_str("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_48_turnconformance_replay_tool_success() { +async fn test_vector_49_turnconformance_replay_tool_success() { let vector: Value = serde_json::from_str("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_49_turnconformance_run_assistant_tool_calls_metadata() { +async fn test_vector_50_turnconformance_run_assistant_tool_calls_metadata() { let vector: Value = serde_json::from_str("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_50_turnconformance_run_async_tool_function() { +async fn test_vector_51_turnconformance_run_async_tool_function() { let vector: Value = serde_json::from_str("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_51_turnconformance_run_bindings_injected() { +async fn test_vector_52_turnconformance_run_bindings_injected() { let vector: Value = serde_json::from_str("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_52_turnconformance_run_cancellation_before_llm() { +async fn test_vector_53_turnconformance_run_cancellation_before_llm() { let vector: Value = serde_json::from_str("{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_53_turnconformance_run_cancellation_between_iterations() { +async fn test_vector_54_turnconformance_run_cancellation_between_iterations() { let vector: Value = serde_json::from_str("{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_54_turnconformance_run_cancellation_between_tools() { +async fn test_vector_55_turnconformance_run_cancellation_between_tools() { let vector: Value = serde_json::from_str("{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_55_turnconformance_run_context_no_trim_when_fits() { +async fn test_vector_56_turnconformance_run_context_no_trim_when_fits() { let vector: Value = serde_json::from_str("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_56_turnconformance_run_context_preserves_system_messages() { +async fn test_vector_57_turnconformance_run_context_preserves_system_messages() { let vector: Value = serde_json::from_str("{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_57_turnconformance_run_context_trim_basic() { +async fn test_vector_58_turnconformance_run_context_trim_basic() { let vector: Value = serde_json::from_str("{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_58_turnconformance_run_empty_tool_result() { +async fn test_vector_59_turnconformance_run_empty_tool_result() { let vector: Value = serde_json::from_str("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_59_turnconformance_run_events_basic_tool_loop() { +async fn test_vector_60_turnconformance_run_events_basic_tool_loop() { let vector: Value = serde_json::from_str("{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_60_turnconformance_run_events_error_logged() { +async fn test_vector_61_turnconformance_run_events_error_logged() { let vector: Value = serde_json::from_str("{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_61_turnconformance_run_events_no_tools() { +async fn test_vector_62_turnconformance_run_events_no_tools() { let vector: Value = serde_json::from_str("{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_62_turnconformance_run_guardrail_all_pass() { +async fn test_vector_63_turnconformance_run_guardrail_all_pass() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_63_turnconformance_run_guardrail_input_deny() { +async fn test_vector_64_turnconformance_run_guardrail_input_deny() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_64_turnconformance_run_guardrail_output_deny() { +async fn test_vector_65_turnconformance_run_guardrail_output_deny() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_65_turnconformance_run_guardrail_tool_deny() { +async fn test_vector_66_turnconformance_run_guardrail_tool_deny() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_66_turnconformance_run_max_iterations_exceeded() { +async fn test_vector_67_turnconformance_run_max_iterations_exceeded() { let vector: Value = serde_json::from_str("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_67_turnconformance_run_multi_turn_tool_calls() { +async fn test_vector_68_turnconformance_run_multi_turn_tool_calls() { let vector: Value = serde_json::from_str("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_68_turnconformance_run_multiple_tool_calls_single_turn() { +async fn test_vector_69_turnconformance_run_multiple_tool_calls_single_turn() { let vector: Value = serde_json::from_str("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_69_turnconformance_run_no_tool_calls() { +async fn test_vector_70_turnconformance_run_no_tool_calls() { let vector: Value = serde_json::from_str("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_70_turnconformance_run_parallel_tools_basic() { +async fn test_vector_71_turnconformance_run_parallel_tools_basic() { let vector: Value = serde_json::from_str("{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_71_turnconformance_run_parallel_tools_with_guardrail_deny() { +async fn test_vector_72_turnconformance_run_parallel_tools_with_guardrail_deny() { let vector: Value = serde_json::from_str("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_72_turnconformance_run_single_tool_call() { +async fn test_vector_73_turnconformance_run_single_tool_call() { let vector: Value = serde_json::from_str("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_73_turnconformance_run_steering_inject_message() { +async fn test_vector_74_turnconformance_run_steering_inject_message() { let vector: Value = serde_json::from_str("{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_74_turnconformance_run_steering_multiple_messages() { +async fn test_vector_75_turnconformance_run_steering_multiple_messages() { let vector: Value = serde_json::from_str("{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_75_turnconformance_run_tool_not_registered_error() { +async fn test_vector_76_turnconformance_run_tool_not_registered_error() { let vector: Value = serde_json::from_str("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_76_turnconformance_run_tool_result_message_format() { +async fn test_vector_77_turnconformance_run_tool_result_message_format() { let vector: Value = serde_json::from_str("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_77_turnconformance_runturn_cancel_before_context() { +async fn test_vector_78_turnconformance_runturn_cancel_before_context() { let vector: Value = serde_json::from_str("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_78_turnconformance_runturn_delegated_provider_state() { +async fn test_vector_79_turnconformance_runturn_delegated_provider_state() { let vector: Value = serde_json::from_str("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_79_turnconformance_runturn_final_output() { +async fn test_vector_80_turnconformance_runturn_final_output() { let vector: Value = serde_json::from_str("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_80_turnconformance_runturn_ordered_tool_round() { +async fn test_vector_81_turnconformance_runturn_ordered_tool_round() { let vector: Value = serde_json::from_str("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_81_turnconformance_runturn_permission_denial_is_model_visible() { +async fn test_vector_82_turnconformance_runturn_permission_denial_is_model_visible() { let vector: Value = serde_json::from_str("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_82_wireconformance_torequest_anthropic_image_format() { +async fn test_vector_83_wireconformance_torequest_anthropic_image_format() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_83_wireconformance_torequest_anthropic_max_tokens_required() { +async fn test_vector_84_wireconformance_torequest_anthropic_max_tokens_required() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_84_wireconformance_torequest_anthropic_options() { +async fn test_vector_85_wireconformance_torequest_anthropic_options() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_85_wireconformance_torequest_anthropic_system_separate() { +async fn test_vector_86_wireconformance_torequest_anthropic_system_separate() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_86_wireconformance_torequest_anthropic_tool_wire() { +async fn test_vector_87_wireconformance_torequest_anthropic_tool_wire() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_87_wireconformance_torequest_chat_audio_mp3() { +async fn test_vector_88_wireconformance_torequest_chat_audio_mp3() { let vector: Value = serde_json::from_str("{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_88_wireconformance_torequest_chat_audio_part() { +async fn test_vector_89_wireconformance_torequest_chat_audio_part() { let vector: Value = serde_json::from_str("{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_89_wireconformance_torequest_chat_image_base64() { +async fn test_vector_90_wireconformance_torequest_chat_image_base64() { let vector: Value = serde_json::from_str("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_90_wireconformance_torequest_chat_image_part() { +async fn test_vector_91_wireconformance_torequest_chat_image_part() { let vector: Value = serde_json::from_str("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_91_wireconformance_torequest_chat_multipart_content() { +async fn test_vector_92_wireconformance_torequest_chat_multipart_content() { let vector: Value = serde_json::from_str("{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_92_wireconformance_torequest_chat_simple() { +async fn test_vector_93_wireconformance_torequest_chat_simple() { let vector: Value = serde_json::from_str("{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_93_wireconformance_torequest_chat_single_text_optimized() { +async fn test_vector_94_wireconformance_torequest_chat_single_text_optimized() { let vector: Value = serde_json::from_str("{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_94_wireconformance_torequest_chat_with_options() { +async fn test_vector_95_wireconformance_torequest_chat_with_options() { let vector: Value = serde_json::from_str("{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_95_wireconformance_torequest_embedding_wire() { +async fn test_vector_96_wireconformance_torequest_embedding_wire() { let vector: Value = serde_json::from_str("{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_96_wireconformance_torequest_image_wire() { +async fn test_vector_97_wireconformance_torequest_image_wire() { let vector: Value = serde_json::from_str("{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_97_wireconformance_torequest_kind_to_json_type_mapping() { +async fn test_vector_98_wireconformance_torequest_kind_to_json_type_mapping() { let vector: Value = serde_json::from_str("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_98_wireconformance_torequest_options_additional_properties() { +async fn test_vector_99_wireconformance_torequest_options_additional_properties() { let vector: Value = serde_json::from_str("{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_99_wireconformance_torequest_options_max_completion_tokens() { +async fn test_vector_100_wireconformance_torequest_options_max_completion_tokens() { let vector: Value = serde_json::from_str("{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_100_wireconformance_torequest_options_stop_sequences() { +async fn test_vector_101_wireconformance_torequest_options_stop_sequences() { let vector: Value = serde_json::from_str("{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_101_wireconformance_torequest_responses_simple() { +async fn test_vector_102_wireconformance_torequest_responses_simple() { let vector: Value = serde_json::from_str("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_102_wireconformance_torequest_responses_structured_output() { +async fn test_vector_103_wireconformance_torequest_responses_structured_output() { let vector: Value = serde_json::from_str("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_103_wireconformance_torequest_responses_with_tools() { +async fn test_vector_104_wireconformance_torequest_responses_with_tools() { let vector: Value = serde_json::from_str("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_104_wireconformance_torequest_structured_output() { +async fn test_vector_105_wireconformance_torequest_structured_output() { let vector: Value = serde_json::from_str("{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_105_wireconformance_torequest_structured_output_nested_optional() { +async fn test_vector_106_wireconformance_torequest_structured_output_nested_optional() { let vector: Value = serde_json::from_str("{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_106_wireconformance_torequest_tools_bindings_stripped() { +async fn test_vector_107_wireconformance_torequest_tools_bindings_stripped() { let vector: Value = serde_json::from_str("{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_107_wireconformance_torequest_tools_function_wire() { +async fn test_vector_108_wireconformance_torequest_tools_function_wire() { let vector: Value = serde_json::from_str("{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_108_wireconformance_torequest_tools_null_when_empty() { +async fn test_vector_109_wireconformance_torequest_tools_null_when_empty() { let vector: Value = serde_json::from_str("{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_109_wireconformance_torequest_tools_strict_mode() { +async fn test_vector_110_wireconformance_torequest_tools_strict_mode() { let vector: Value = serde_json::from_str("{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; diff --git a/runtime/swift/prompty-model/Sources/PromptyModel/live_chat_conformance.swift b/runtime/swift/prompty-model/Sources/PromptyModel/live_chat_conformance.swift new file mode 100644 index 000000000..2083e2ca3 --- /dev/null +++ b/runtime/swift/prompty-model/Sources/PromptyModel/live_chat_conformance.swift @@ -0,0 +1,8 @@ +// +// Code generated by Typra emitter; DO NOT EDIT. + +import Foundation + +public protocol LiveChatConformance: Sendable { + func complete(input: Any) async throws -> Any +} diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift index cbc614601..6f04efdd1 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift @@ -40,7 +40,10 @@ enum VectorAdapters { } static func waivers() -> [String: String] { - return [:] + return [ + "LiveChatConformance.complete": + "drill: consumer-owned transport-seam probe (Python/TS only)" + ] } static func doubles() -> Any? { diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift index dbeb6e291..c89f0b2bd 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift @@ -194,7 +194,17 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "DiscoveryConformance", operation: "mapModel", vector: vector, sync: false, seam: seam()) } - func testVector16LoadConformanceLoadBasicLoad() async throws { + func testVector16LiveChatConformanceCompleteOpenaiChatDrill() async throws { + guard let vectorData = "{\"name\":\"openai_chat_drill\",\"stage\":\"drill\",\"provider\":\"openai\",\"input\":{\"provider\":\"openai\",\"model\":\"gpt-4o-mini\",\"apiKey\":{\"$env\":\"OPENAI_API_KEY\"},\"endpoint\":{\"$env\":\"OPENAI_BASE_URL\"},\"messages\":[{\"role\":\"user\",\"content\":\"Say hello in exactly one word.\"}],\"options\":{\"temperature\":0,\"maxOutputTokens\":16}},\"exchange\":{\"transport\":{\"baseUrl\":{\"$env\":\"OPENAI_BASE_URL\"}},\"cassette\":{\"$json\":\"cassettes/openai_chat_drill.json\"},\"planes\":[\"transport\",\"wire\",\"semantic\"]},\"expected\":{\"role\":\"assistant\",\"contentNonEmpty\":true,\"finishReasonInEnum\":true},\"operation\":\"complete\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LiveChatConformance", operation: "complete", vector: vector, sync: false, seam: seam()) + } + + func testVector17LoadConformanceLoadBasicLoad() async throws { guard let vectorData = "{\"name\":\"basic_load\",\"description\":\"Load basic.prompty with env var resolution and verify all PromptAgent fields\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"basic-prompt\",\"description\":\"A basic prompt for testing\",\"metadata\":{\"authors\":[\"testauthor\"]},\"model\":{\"id\":\"gpt-4\",\"provider\":\"openai\",\"apiType\":\"chat\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"},\"options\":{\"temperature\":0.7,\"maxOutputTokens\":1000}},\"inputs\":[{\"name\":\"firstName\",\"kind\":\"string\",\"default\":\"Jane\"},{\"name\":\"lastName\",\"kind\":\"string\",\"default\":\"Doe\"},{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What is the meaning of life?\"}],\"template\":{\"format\":{\"kind\":\"jinja2\"},\"parser\":{\"kind\":\"prompty\"}},\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -204,7 +214,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector17LoadConformanceLoadConnectionTypesLoad() async throws { + func testVector18LoadConformanceLoadConnectionTypesLoad() async throws { guard let vectorData = "{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -214,7 +224,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector18LoadConformanceLoadEmbeddingLoad() async throws { + func testVector19LoadConformanceLoadEmbeddingLoad() async throws { guard let vectorData = "{\"name\":\"embedding_load\",\"description\":\"Load embedding.prompty and verify apiType=embedding with correct model id\",\"stage\":\"load\",\"input\":{\"fixture\":\"embedding.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"embedding\",\"model\":{\"id\":\"text-embedding-3-small\",\"provider\":\"openai\",\"apiType\":\"embedding\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"}},\"inputs\":[{\"name\":\"text\",\"kind\":\"string\",\"default\":\"Hello world\"}],\"instructions\":\"{{text}}\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -224,7 +234,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector19LoadConformanceLoadEmptyFrontmatterBodyOnly() async throws { + func testVector20LoadConformanceLoadEmptyFrontmatterBodyOnly() async throws { guard let vectorData = "{\"name\":\"empty_frontmatter_body_only\",\"description\":\"Empty frontmatter with body \u{2014} instructions extracted, defaults applied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"empty-fm\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"empty-fm\",\"model\":null,\"inputs\":null,\"tools\":null},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -234,7 +244,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector20LoadConformanceLoadEnvDefault() async throws { + func testVector21LoadConformanceLoadEnvDefault() async throws { guard let vectorData = "{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -244,7 +254,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector21LoadConformanceLoadEnvMissingError() async throws { + func testVector22LoadConformanceLoadEnvMissingError() async throws { guard let vectorData = "{\"name\":\"env_missing_error\",\"description\":\"${env:NONEXISTENT} with no default and var not set raises an error\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-error-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:NONEXISTENT}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"env_var_not_set\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -254,7 +264,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector22LoadConformanceLoadEnvResolution() async throws { + func testVector23LoadConformanceLoadEnvResolution() async throws { guard let vectorData = "{\"name\":\"env_resolution\",\"description\":\"${env:MY_VAR} resolves to the environment variable value when set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MY_VAR}\"}}},\"env\":{\"MY_VAR\":\"hello\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"hello\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -264,7 +274,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector23LoadConformanceLoadFileReferenceInTreeAllowed() async throws { + func testVector24LoadConformanceLoadFileReferenceInTreeAllowed() async throws { guard let vectorData = "{\"name\":\"file_reference_in_tree_allowed\",\"description\":\"A ${file:sub/data.json} reference that stays within the prompt file's directory tree resolves to the parsed content (containment control alongside the traversal-reject vector)\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-in-tree-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:sub/data.json}\"}},\"files\":{\"sub/data.json\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\",\"apiKey\":\"ok\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -274,7 +284,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector24LoadConformanceLoadFileReferenceTraversalRejected() async throws { + func testVector25LoadConformanceLoadFileReferenceTraversalRejected() async throws { guard let vectorData = "{\"name\":\"file_reference_traversal_rejected\",\"description\":\"A ${file:../secret.json} reference that escapes the prompt file's directory via `..` is rejected even though the target file exists, preventing arbitrary file disclosure (GHSA-7pfc-5v9r-j299). Uses forward-slash separators so the vector holds on every OS.\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-traversal-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:../secret.json}\"}},\"files\":{\"../secret.json\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"leaked\"}}},\"expectedError\":{\"kind\":\"file_reference\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -284,7 +294,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector25LoadConformanceLoadFileResolution() async throws { + func testVector26LoadConformanceLoadFileResolution() async throws { guard let vectorData = "{\"name\":\"file_resolution\",\"description\":\"${file:shared_connection.json} resolves to the parsed JSON content of the referenced file\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"file-ref-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:shared_connection.json}\"}},\"env\":{\"OPENAI_API_KEY\":\"shared-key\"},\"files\":{\"shared_connection.json\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\",\"apiKey\":\"shared-key\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -294,7 +304,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector26LoadConformanceLoadImageApitypeLoad() async throws { + func testVector27LoadConformanceLoadImageApitypeLoad() async throws { guard let vectorData = "{\"name\":\"image_apitype_load\",\"description\":\"An image-generation prompt loads with apiType image and passthrough additionalProperties model options\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}}},\"expected\":{\"kind\":\"prompt\",\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -304,7 +314,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector27LoadConformanceLoadInputScalarShorthand() async throws { + func testVector28LoadConformanceLoadInputScalarShorthand() async throws { guard let vectorData = "{\"name\":\"input_scalar_shorthand\",\"description\":\"Scalar input values create typed Property with default set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"scalar-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]}},\"expected\":{\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -314,7 +324,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector28LoadConformanceLoadInputValidationDefaultFill() async throws { + func testVector29LoadConformanceLoadInputValidationDefaultFill() async throws { guard let vectorData = "{\"name\":\"input_validation_default_fill\",\"description\":\"An input with a default value is filled in when no value is provided\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"default-fill-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"name\",\"kind\":\"string\",\"default\":\"world\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{\"name\":\"world\"}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -324,7 +334,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector29LoadConformanceLoadInputValidationExampleNotUsed() async throws { + func testVector30LoadConformanceLoadInputValidationExampleNotUsed() async throws { guard let vectorData = "{\"name\":\"input_validation_example_not_used\",\"description\":\"An input with example='sample' but no default must NOT use the example at runtime\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"example-not-used-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"required\":false,\"example\":\"sample\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -334,7 +344,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector30LoadConformanceLoadInputValidationOptionalOmit() async throws { + func testVector31LoadConformanceLoadInputValidationOptionalOmit() async throws { guard let vectorData = "{\"name\":\"input_validation_optional_omit\",\"description\":\"A non-required input with no default and no value is omitted (not an error)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"optional-omit-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"nickname\",\"kind\":\"string\",\"required\":false}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -344,7 +354,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector31LoadConformanceLoadInputValidationRequired() async throws { + func testVector32LoadConformanceLoadInputValidationRequired() async throws { guard let vectorData = "{\"name\":\"input_validation_required\",\"description\":\"A required input with no default and no value provided raises ValueError\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"required-input-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},\"inputs\":{}},\"expectedError\":{\"kind\":\"missing_required_input\",\"field\":\"city\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -354,7 +364,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector32LoadConformanceLoadInstructionsFromBody() async throws { + func testVector33LoadConformanceLoadInstructionsFromBody() async throws { guard let vectorData = "{\"name\":\"instructions_from_body\",\"description\":\"The markdown body after the closing --- becomes the instructions field\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -364,7 +374,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector33LoadConformanceLoadInvalidFrontmatterError() async throws { + func testVector34LoadConformanceLoadInvalidFrontmatterError() async throws { guard let vectorData = "{\"name\":\"invalid_frontmatter_error\",\"description\":\"Malformed YAML in frontmatter raises a parse error\",\"stage\":\"load\",\"input\":{\"frontmatter_raw\":\"---\\nname: [invalid\\n---\\nHello\"},\"expectedError\":{\"kind\":\"invalid_frontmatter\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -374,7 +384,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector34LoadConformanceLoadKindAlwaysPrompt() async throws { + func testVector35LoadConformanceLoadKindAlwaysPrompt() async throws { guard let vectorData = "{\"name\":\"kind_always_prompt\",\"description\":\"The loader always injects kind='prompt' \u{2014} .prompty files never specify kind themselves\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -384,7 +394,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector35LoadConformanceLoadMinimalLoad() async throws { + func testVector36LoadConformanceLoadMinimalLoad() async throws { guard let vectorData = "{\"name\":\"minimal_load\",\"description\":\"Load minimal.prompty with model shorthand and bare-minimum frontmatter\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"minimal\",\"model\":{\"id\":\"gpt-4\"},\"instructions\":\"system:\\nHello world.\",\"inputs\":null,\"outputs\":null,\"tools\":null},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -394,7 +404,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector36LoadConformanceLoadMissingFileError() async throws { + func testVector37LoadConformanceLoadMissingFileError() async throws { guard let vectorData = "{\"name\":\"missing_file_error\",\"description\":\"Loading a nonexistent .prompty file raises FileNotFoundError\",\"stage\":\"load\",\"input\":{\"fixture\":\"nonexistent.prompty\"},\"expectedError\":{\"kind\":\"file_not_found\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -404,7 +414,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector37LoadConformanceLoadModelShorthand() async throws { + func testVector38LoadConformanceLoadModelShorthand() async throws { guard let vectorData = "{\"name\":\"model_shorthand\",\"description\":\"String shorthand 'model: gpt-4o' expands to a Model object with id set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"test\",\"model\":\"gpt-4o\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"test\",\"model\":{\"id\":\"gpt-4o\"}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -414,7 +424,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector38LoadConformanceLoadStructuredOutputsLoad() async throws { + func testVector39LoadConformanceLoadStructuredOutputsLoad() async throws { guard let vectorData = "{\"name\":\"structured_outputs_load\",\"description\":\"Outputs load as a list of properties preserving kind, description, and required flags\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"structured-output\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]}},\"expected\":{\"kind\":\"prompt\",\"name\":\"structured-output\",\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -424,7 +434,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector39LoadConformanceLoadTemplateStringInvalid() async throws { + func testVector40LoadConformanceLoadTemplateStringInvalid() async throws { guard let vectorData = "{\"name\":\"template_string_invalid\",\"description\":\"Template as a bare string is not valid v2 \u{2014} must be an object with format/parser\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"shorthand-test\",\"model\":{\"id\":\"gpt-4\"},\"template\":\"jinja2\"}},\"expectedError\":{\"kind\":\"invalid_template\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -434,7 +444,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector40LoadConformanceLoadToolsCustomLoad() async throws { + func testVector41LoadConformanceLoadToolsCustomLoad() async throws { guard let vectorData = "{\"name\":\"tools_custom_load\",\"description\":\"Load a prompty with an unknown tool kind \u{2014} falls through to CustomTool\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"custom-tool-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://custom.example.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\"}]},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -444,7 +454,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector41LoadConformanceLoadToolsFunctionLoad() async throws { + func testVector42LoadConformanceLoadToolsFunctionLoad() async throws { guard let vectorData = "{\"name\":\"tools_function_load\",\"description\":\"Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings\",\"stage\":\"load\",\"input\":{\"fixture\":\"tools_function.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"function-tools\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather in a given location\",\"strict\":true,\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"description\":\"The city and state, e.g. San Francisco, CA\"},{\"name\":\"unit\",\"kind\":\"string\",\"description\":\"Temperature unit\",\"enumValues\":[\"celsius\",\"fahrenheit\"]}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"inputs\":[{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What's the weather in Paris?\"},{\"name\":\"preferred_unit\",\"kind\":\"string\",\"default\":\"celsius\"}],\"instructions\":\"system:\\nYou are a helpful assistant with access to tools.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -454,7 +464,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector42LoadConformanceLoadToolsMcpLoad() async throws { + func testVector43LoadConformanceLoadToolsMcpLoad() async throws { guard let vectorData = "{\"name\":\"tools_mcp_load\",\"description\":\"Load a prompty with an MCP tool \u{2014} kind, serverName, connection preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"mcp-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-mcp\"}}]}},\"expected\":{\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\"}]},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -464,7 +474,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector43LoadConformanceLoadToolsOpenapiLoad() async throws { + func testVector44LoadConformanceLoadToolsOpenapiLoad() async throws { guard let vectorData = "{\"name\":\"tools_openapi_load\",\"description\":\"Load a prompty with an OpenAPI tool \u{2014} specification path preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"openapi-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://api.weather.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\"}]},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -474,7 +484,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector44TurnConformanceReplayMaxIterations() async throws { + func testVector45TurnConformanceReplayMaxIterations() async throws { guard let vectorData = "{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -484,7 +494,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector45TurnConformanceReplayNoTool() async throws { + func testVector46TurnConformanceReplayNoTool() async throws { guard let vectorData = "{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -494,7 +504,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector46TurnConformanceReplayPermissionDenied() async throws { + func testVector47TurnConformanceReplayPermissionDenied() async throws { guard let vectorData = "{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -504,7 +514,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector47TurnConformanceReplayToolFailure() async throws { + func testVector48TurnConformanceReplayToolFailure() async throws { guard let vectorData = "{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -514,7 +524,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector48TurnConformanceReplayToolSuccess() async throws { + func testVector49TurnConformanceReplayToolSuccess() async throws { guard let vectorData = "{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -524,7 +534,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector49TurnConformanceRunAssistantToolCallsMetadata() async throws { + func testVector50TurnConformanceRunAssistantToolCallsMetadata() async throws { guard let vectorData = "{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -534,7 +544,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector50TurnConformanceRunAsyncToolFunction() async throws { + func testVector51TurnConformanceRunAsyncToolFunction() async throws { guard let vectorData = "{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -544,7 +554,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector51TurnConformanceRunBindingsInjected() async throws { + func testVector52TurnConformanceRunBindingsInjected() async throws { guard let vectorData = "{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -554,7 +564,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector52TurnConformanceRunCancellationBeforeLlm() async throws { + func testVector53TurnConformanceRunCancellationBeforeLlm() async throws { guard let vectorData = "{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -564,7 +574,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector53TurnConformanceRunCancellationBetweenIterations() async throws { + func testVector54TurnConformanceRunCancellationBetweenIterations() async throws { guard let vectorData = "{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -574,7 +584,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector54TurnConformanceRunCancellationBetweenTools() async throws { + func testVector55TurnConformanceRunCancellationBetweenTools() async throws { guard let vectorData = "{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -584,7 +594,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector55TurnConformanceRunContextNoTrimWhenFits() async throws { + func testVector56TurnConformanceRunContextNoTrimWhenFits() async throws { guard let vectorData = "{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -594,7 +604,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector56TurnConformanceRunContextPreservesSystemMessages() async throws { + func testVector57TurnConformanceRunContextPreservesSystemMessages() async throws { guard let vectorData = "{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -604,7 +614,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector57TurnConformanceRunContextTrimBasic() async throws { + func testVector58TurnConformanceRunContextTrimBasic() async throws { guard let vectorData = "{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -614,7 +624,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector58TurnConformanceRunEmptyToolResult() async throws { + func testVector59TurnConformanceRunEmptyToolResult() async throws { guard let vectorData = "{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -624,7 +634,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector59TurnConformanceRunEventsBasicToolLoop() async throws { + func testVector60TurnConformanceRunEventsBasicToolLoop() async throws { guard let vectorData = "{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -634,7 +644,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector60TurnConformanceRunEventsErrorLogged() async throws { + func testVector61TurnConformanceRunEventsErrorLogged() async throws { guard let vectorData = "{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -644,7 +654,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector61TurnConformanceRunEventsNoTools() async throws { + func testVector62TurnConformanceRunEventsNoTools() async throws { guard let vectorData = "{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -654,7 +664,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector62TurnConformanceRunGuardrailAllPass() async throws { + func testVector63TurnConformanceRunGuardrailAllPass() async throws { guard let vectorData = "{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -664,7 +674,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector63TurnConformanceRunGuardrailInputDeny() async throws { + func testVector64TurnConformanceRunGuardrailInputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -674,7 +684,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector64TurnConformanceRunGuardrailOutputDeny() async throws { + func testVector65TurnConformanceRunGuardrailOutputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -684,7 +694,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector65TurnConformanceRunGuardrailToolDeny() async throws { + func testVector66TurnConformanceRunGuardrailToolDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -694,7 +704,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector66TurnConformanceRunMaxIterationsExceeded() async throws { + func testVector67TurnConformanceRunMaxIterationsExceeded() async throws { guard let vectorData = "{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -704,7 +714,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector67TurnConformanceRunMultiTurnToolCalls() async throws { + func testVector68TurnConformanceRunMultiTurnToolCalls() async throws { guard let vectorData = "{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -714,7 +724,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector68TurnConformanceRunMultipleToolCallsSingleTurn() async throws { + func testVector69TurnConformanceRunMultipleToolCallsSingleTurn() async throws { guard let vectorData = "{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -724,7 +734,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector69TurnConformanceRunNoToolCalls() async throws { + func testVector70TurnConformanceRunNoToolCalls() async throws { guard let vectorData = "{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -734,7 +744,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector70TurnConformanceRunParallelToolsBasic() async throws { + func testVector71TurnConformanceRunParallelToolsBasic() async throws { guard let vectorData = "{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -744,7 +754,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector71TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { + func testVector72TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -754,7 +764,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector72TurnConformanceRunSingleToolCall() async throws { + func testVector73TurnConformanceRunSingleToolCall() async throws { guard let vectorData = "{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -764,7 +774,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector73TurnConformanceRunSteeringInjectMessage() async throws { + func testVector74TurnConformanceRunSteeringInjectMessage() async throws { guard let vectorData = "{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -774,7 +784,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector74TurnConformanceRunSteeringMultipleMessages() async throws { + func testVector75TurnConformanceRunSteeringMultipleMessages() async throws { guard let vectorData = "{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -784,7 +794,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector75TurnConformanceRunToolNotRegisteredError() async throws { + func testVector76TurnConformanceRunToolNotRegisteredError() async throws { guard let vectorData = "{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -794,7 +804,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector76TurnConformanceRunToolResultMessageFormat() async throws { + func testVector77TurnConformanceRunToolResultMessageFormat() async throws { guard let vectorData = "{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -804,7 +814,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector77TurnConformanceRunTurnCancelBeforeContext() async throws { + func testVector78TurnConformanceRunTurnCancelBeforeContext() async throws { guard let vectorData = "{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -814,7 +824,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector78TurnConformanceRunTurnDelegatedProviderState() async throws { + func testVector79TurnConformanceRunTurnDelegatedProviderState() async throws { guard let vectorData = "{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -824,7 +834,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector79TurnConformanceRunTurnFinalOutput() async throws { + func testVector80TurnConformanceRunTurnFinalOutput() async throws { guard let vectorData = "{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -834,7 +844,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector80TurnConformanceRunTurnOrderedToolRound() async throws { + func testVector81TurnConformanceRunTurnOrderedToolRound() async throws { guard let vectorData = "{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -844,7 +854,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector81TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { + func testVector82TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { guard let vectorData = "{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -854,7 +864,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector82WireConformanceToRequestAnthropicImageFormat() async throws { + func testVector83WireConformanceToRequestAnthropicImageFormat() async throws { guard let vectorData = "{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -864,7 +874,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector83WireConformanceToRequestAnthropicMaxTokensRequired() async throws { + func testVector84WireConformanceToRequestAnthropicMaxTokensRequired() async throws { guard let vectorData = "{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -874,7 +884,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector84WireConformanceToRequestAnthropicOptions() async throws { + func testVector85WireConformanceToRequestAnthropicOptions() async throws { guard let vectorData = "{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -884,7 +894,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector85WireConformanceToRequestAnthropicSystemSeparate() async throws { + func testVector86WireConformanceToRequestAnthropicSystemSeparate() async throws { guard let vectorData = "{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -894,7 +904,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector86WireConformanceToRequestAnthropicToolWire() async throws { + func testVector87WireConformanceToRequestAnthropicToolWire() async throws { guard let vectorData = "{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -904,7 +914,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector87WireConformanceToRequestChatAudioMp3() async throws { + func testVector88WireConformanceToRequestChatAudioMp3() async throws { guard let vectorData = "{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -914,7 +924,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector88WireConformanceToRequestChatAudioPart() async throws { + func testVector89WireConformanceToRequestChatAudioPart() async throws { guard let vectorData = "{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -924,7 +934,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector89WireConformanceToRequestChatImageBase64() async throws { + func testVector90WireConformanceToRequestChatImageBase64() async throws { guard let vectorData = "{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -934,7 +944,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector90WireConformanceToRequestChatImagePart() async throws { + func testVector91WireConformanceToRequestChatImagePart() async throws { guard let vectorData = "{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -944,7 +954,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector91WireConformanceToRequestChatMultipartContent() async throws { + func testVector92WireConformanceToRequestChatMultipartContent() async throws { guard let vectorData = "{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -954,7 +964,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector92WireConformanceToRequestChatSimple() async throws { + func testVector93WireConformanceToRequestChatSimple() async throws { guard let vectorData = "{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -964,7 +974,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector93WireConformanceToRequestChatSingleTextOptimized() async throws { + func testVector94WireConformanceToRequestChatSingleTextOptimized() async throws { guard let vectorData = "{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -974,7 +984,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector94WireConformanceToRequestChatWithOptions() async throws { + func testVector95WireConformanceToRequestChatWithOptions() async throws { guard let vectorData = "{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -984,7 +994,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector95WireConformanceToRequestEmbeddingWire() async throws { + func testVector96WireConformanceToRequestEmbeddingWire() async throws { guard let vectorData = "{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -994,7 +1004,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector96WireConformanceToRequestImageWire() async throws { + func testVector97WireConformanceToRequestImageWire() async throws { guard let vectorData = "{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1004,7 +1014,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector97WireConformanceToRequestKindToJsonTypeMapping() async throws { + func testVector98WireConformanceToRequestKindToJsonTypeMapping() async throws { guard let vectorData = "{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1014,7 +1024,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector98WireConformanceToRequestOptionsAdditionalProperties() async throws { + func testVector99WireConformanceToRequestOptionsAdditionalProperties() async throws { guard let vectorData = "{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1024,7 +1034,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector99WireConformanceToRequestOptionsMaxCompletionTokens() async throws { + func testVector100WireConformanceToRequestOptionsMaxCompletionTokens() async throws { guard let vectorData = "{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1034,7 +1044,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector100WireConformanceToRequestOptionsStopSequences() async throws { + func testVector101WireConformanceToRequestOptionsStopSequences() async throws { guard let vectorData = "{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1044,7 +1054,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector101WireConformanceToRequestResponsesSimple() async throws { + func testVector102WireConformanceToRequestResponsesSimple() async throws { guard let vectorData = "{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1054,7 +1064,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector102WireConformanceToRequestResponsesStructuredOutput() async throws { + func testVector103WireConformanceToRequestResponsesStructuredOutput() async throws { guard let vectorData = "{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1064,7 +1074,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector103WireConformanceToRequestResponsesWithTools() async throws { + func testVector104WireConformanceToRequestResponsesWithTools() async throws { guard let vectorData = "{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1074,7 +1084,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector104WireConformanceToRequestStructuredOutput() async throws { + func testVector105WireConformanceToRequestStructuredOutput() async throws { guard let vectorData = "{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1084,7 +1094,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector105WireConformanceToRequestStructuredOutputNestedOptional() async throws { + func testVector106WireConformanceToRequestStructuredOutputNestedOptional() async throws { guard let vectorData = "{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1094,7 +1104,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector106WireConformanceToRequestToolsBindingsStripped() async throws { + func testVector107WireConformanceToRequestToolsBindingsStripped() async throws { guard let vectorData = "{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1104,7 +1114,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector107WireConformanceToRequestToolsFunctionWire() async throws { + func testVector108WireConformanceToRequestToolsFunctionWire() async throws { guard let vectorData = "{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1114,7 +1124,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector108WireConformanceToRequestToolsNullWhenEmpty() async throws { + func testVector109WireConformanceToRequestToolsNullWhenEmpty() async throws { guard let vectorData = "{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1124,7 +1134,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector109WireConformanceToRequestToolsStrictMode() async throws { + func testVector110WireConformanceToRequestToolsStrictMode() async throws { guard let vectorData = "{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { diff --git a/runtime/typescript/packages/core/src/model/index.ts b/runtime/typescript/packages/core/src/model/index.ts index 5ca5aa4f0..4e1bf721d 100644 --- a/runtime/typescript/packages/core/src/model/index.ts +++ b/runtime/typescript/packages/core/src/model/index.ts @@ -14,6 +14,7 @@ export type { EventJournalWriter } from "./event-journal-writer"; export type { EventSink } from "./event-sink"; export type { Executor } from "./executor"; export type { HostToolExecutor } from "./host-tool-executor"; +export type { LiveChatConformance } from "./live-chat-conformance"; export type { LoadConformance } from "./load-conformance"; export type { ModelLister } from "./model-lister"; export type { Parser } from "./parser"; diff --git a/runtime/typescript/packages/core/src/model/live-chat-conformance.ts b/runtime/typescript/packages/core/src/model/live-chat-conformance.ts new file mode 100644 index 000000000..365d29aab --- /dev/null +++ b/runtime/typescript/packages/core/src/model/live-chat-conformance.ts @@ -0,0 +1,15 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// WARNING: This is an auto-generated file. DO NOT EDIT THIS FILE DIRECTLY. + +/** Conformance-only seam hosting the "service drill" vectors: a conformance +vector that crosses the REAL Executor->Processor transport seam instead of +stopping at the SDK edge. A drill runs against a live provider when +credentials are present and REPLAYS a recorded cassette when they are not; +the live<->replay parity is the gate. + +Like every other conformance seam this interface exists purely to carry */ +export interface LiveChatConformance { + /** Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes */ + complete(input: unknown): Promise; +} diff --git a/runtime/typescript/packages/core/tests/model/cassettes/openai_chat_drill.json b/runtime/typescript/packages/core/tests/model/cassettes/openai_chat_drill.json new file mode 100644 index 000000000..8afdc52e0 --- /dev/null +++ b/runtime/typescript/packages/core/tests/model/cassettes/openai_chat_drill.json @@ -0,0 +1,44 @@ +{ + "_comment": "Service-drill cassette for LiveChatConformance.complete/openai_chat_drill. Consumer-owned roster: NOT modeled in TypeSpec/Typra. Regenerate with `uv run python tests/model/record_cassette.py`. `request.body` is the wire plane the drill adapter asserts; `response.body` is replayed as the seam bytes.", + "recordedAt": "2026-09-04T22:41:20Z", + "request": { + "body": { + "max_completion_tokens": 16, + "messages": [ + { + "content": "Say hello in exactly one word.", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "temperature": 0 + }, + "method": "POST", + "path": "/chat/completions" + }, + "response": { + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "Hello", + "role": "assistant" + } + } + ], + "created": 1700000000, + "id": "chatcmpl-drill-openai-0001", + "model": "gpt-4o-mini", + "object": "chat.completion", + "usage": { + "completion_tokens": 1, + "prompt_tokens": 12, + "total_tokens": 13 + } + }, + "status": 200 + } +} diff --git a/runtime/typescript/packages/core/tests/model/drill-replay.ts b/runtime/typescript/packages/core/tests/model/drill-replay.ts new file mode 100644 index 000000000..05357bdda --- /dev/null +++ b/runtime/typescript/packages/core/tests/model/drill-replay.ts @@ -0,0 +1,94 @@ +/** + * Local cassette-replay HTTP transport for the service-drill vector. + * + * The drill binds the Executor->Processor transport seam via **base-URL + * redirect**: the provider SDK is pointed at a local `node:http` server that + * serves a recorded response and records the inbound request. This is the + * consumer-owned "roster" half of the drill -- deliberately NOT modeled in + * TypeSpec/Typra, since cassette bytes and provider-SDK transport binding stay + * runtime-owned. It is the TypeScript counterpart of the Python runtime's + * `tests/model/_drill_replay.py`. + * + * No production code changes: the executor's own connection `endpoint` is set + * to `server.baseUrl` and its OpenAI SDK POSTs there exactly as it would to the + * real base URL. + */ + +import { createServer, type IncomingMessage, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; + +export type CapturedRequest = { + method: string; + path: string; + body: unknown; +}; + +/** + * A background HTTP server that replays one response and records requests. + * + * `baseUrl` is the redirect target to hand the provider SDK; `lastRequest` is + * the most recently captured outbound request (the wire plane's observed + * value). Use `start()`/`stop()` around the transport-bound invocation. + */ +export class CassetteReplayServer { + readonly captured: CapturedRequest[] = []; + private server: Server | null = null; + + constructor( + private readonly responseBody: unknown, + private readonly responseStatus: number = 200, + ) {} + + async start(): Promise { + const payload = JSON.stringify(this.responseBody); + this.server = createServer((req: IncomingMessage, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + let body: unknown = {}; + if (raw) { + try { + body = JSON.parse(raw); + } catch { + body = { __raw__: raw }; + } + } + this.captured.push({ + method: req.method ?? "POST", + path: req.url ?? "/", + body, + }); + res.writeHead(this.responseStatus, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(payload).toString(), + }); + res.end(payload); + }); + }); + await new Promise((resolveListen) => { + this.server!.listen(0, "127.0.0.1", () => resolveListen()); + }); + } + + async stop(): Promise { + if (!this.server) return; + await new Promise((resolveClose) => { + this.server!.close(() => resolveClose()); + }); + this.server = null; + } + + get baseUrl(): string { + if (!this.server) throw new Error("replay server not started"); + const { address, port } = this.server.address() as AddressInfo; + const host = address === "::" ? "127.0.0.1" : address; + return `http://${host}:${port}`; + } + + get lastRequest(): CapturedRequest | null { + return this.captured.length > 0 + ? this.captured[this.captured.length - 1] + : null; + } +} diff --git a/runtime/typescript/packages/core/tests/model/vector-adapters.ts b/runtime/typescript/packages/core/tests/model/vector-adapters.ts index 5a75d703d..bf9bfe4e8 100644 --- a/runtime/typescript/packages/core/tests/model/vector-adapters.ts +++ b/runtime/typescript/packages/core/tests/model/vector-adapters.ts @@ -81,11 +81,23 @@ import { defaultSaveContext } from "../../src/core/loader.js"; import { TurnOptions } from "../../src/model/index.js"; import { enrich, mapModel } from "../../src/model/discovery.js"; import { ModelInfo } from "../../src/model/contracts/models/model-info.js"; +// Drill transport bind: the OpenAI executor checks `connection instanceof +// ApiKeyConnection` against the built `@prompty/core` package (dist), whose class +// identity differs from the `../../src` core imported above. The drill agent must +// therefore be constructed from the dist package so the executor recognizes its +// key connection. This cross-core seam is a per-runtime roster detail, not an +// emitter concern. +import { + Agent as DrillAgent, + LoadContext as DrillLoadContext, + Message as DrillMessage, +} from "@prompty/core"; import { buildChatArgs as openaiBuildChatArgs, buildEmbeddingArgs as openaiBuildEmbeddingArgs, buildImageArgs as openaiBuildImageArgs, buildResponsesArgs as openaiBuildResponsesArgs, + OpenAIExecutor, OpenAIProcessor, processResponse as openaiProcessResponse, } from "@prompty/openai"; @@ -93,6 +105,7 @@ import { AnthropicProcessor, buildChatArgs as anthropicBuildChatArgs, } from "@prompty/anthropic"; +import { CassetteReplayServer } from "./drill-replay.js"; // The pipeline drives renderer/parser lookups through the registry; register the // built-ins once so the render/parse adapters exercise the real runtime path. @@ -1058,6 +1071,249 @@ async function runTurnInvoke( }; } +// --------------------------------------------------------------------------- +// LiveChatConformance.complete -- the service-drill adapter +// --------------------------------------------------------------------------- +// +// A "drill" is a conformance vector that crosses the REAL Executor transport +// seam. Unlike the pure conformance vectors that stop at the SDK edge, the drill +// binds the seam via base-URL redirect -- to a local `CassetteReplayServer` +// (replay mode) or the real base URL (live mode) -- and asserts three planes: +// +// 1. transport -- the seam was reached and returned bytes (executor got a +// response, and the replay server captured an inbound request). +// 2. wire -- the outbound provider request body matches the cassette's +// recorded request body (canonical equality; throw on mismatch). +// 3. semantic -- normalized executor output == the vector `expected`. This +// plane is asserted by the RUNNER via the registered `normalize`, so the +// drill's semantic plane IS a conformance vector over the executor. +// +// In live mode the adapter additionally asserts live-semantic == replay-semantic +// (parity). `requires` is deliberately OMITTED from the drill vector -- the +// `requires` guard is all-or-nothing and would skip the WHOLE vector when creds +// are absent, defeating the replay plane. Instead the adapter self-gates the +// live plane on credential presence and always runs replay. +// +// Cassette bytes and the SDK transport binding are consumer-owned "roster" -- by +// design NOT modeled in TypeSpec/Typra. This is the TypeScript counterpart of +// the Python runtime's drill adapter in `tests/model/vector_adapters.py`. + +const DRILL_REPLAY_API_KEY = "sk-drill-replay-placeholder"; +const DRILL_FINISH_REASONS = new Set([ + "stop", + "length", + "tool_calls", + "content_filter", + "function_call", +]); + +/** Order-insensitive canonical JSON for wire/parity equality checks. */ +function drillCanonical(value: unknown): string { + const sort = (node: unknown): unknown => { + if (Array.isArray(node)) return node.map(sort); + if (node !== null && typeof node === "object") { + const src = node as Record; + const out: Record = {}; + for (const key of Object.keys(src).sort()) out[key] = sort(src[key]); + return out; + } + return node; + }; + return JSON.stringify(sort(value)); +} + +/** + * Construct a chat `Agent` bound to a specific transport `endpoint`. + * + * The drill binds the transport seam via base-URL redirect, so `endpoint` and + * `apiKey` are supplied by the caller (a local cassette-replay server in replay + * mode, the real base URL + key in live mode) rather than read from the vector + * input. When `endpoint` is falsy the connection omits it so the SDK falls back + * to the provider's default base URL (live mode without an override). + */ +function drillBuildAgent( + input: any, + opts: { endpoint?: string | null; apiKey?: string | null }, +): DrillAgent { + const provider = (input.provider ?? "openai").toLowerCase(); + const connection: Record = { kind: "key" }; + if (opts.apiKey) connection.apiKey = opts.apiKey; + if (opts.endpoint) connection.endpoint = opts.endpoint; + + const model: Record = { + id: input.model ?? "gpt-4o-mini", + provider, + apiType: "chat", + connection, + }; + if (input.options) model.options = input.options; + + return DrillAgent.load( + { name: "drill-chat-vector", model }, + new DrillLoadContext(), + ); +} + +function drillBuildMessages(input: any): DrillMessage[] { + return ((input.messages ?? []) as any[]).map( + (spec) => + new DrillMessage({ + role: spec.role ?? "user", + parts: [{ kind: "text", value: spec.content ?? "" }], + }), + ); +} + +/** + * Drive the real executor with its transport bound to `endpoint`. Returns the + * raw provider response. Shared by the replay path (`endpoint` = a + * `CassetteReplayServer` base URL) and the live path (`endpoint` = the real + * base URL or null for the SDK default). + */ +async function drillExecute( + input: any, + opts: { endpoint?: string | null; apiKey?: string | null }, +): Promise { + const agent = drillBuildAgent(input, opts); + const messages = drillBuildMessages(input); + return new OpenAIExecutor().execute(agent, messages); +} + +/** Reduce a raw provider chat response to `{ role, content, finish }`. */ +function drillExtractChatStructure(observed: any): { + role: unknown; + content: unknown; + finish: unknown; +} { + const choices = observed?.choices; + if (Array.isArray(choices) && choices.length > 0) { + const choice = choices[0]; + const message = choice?.message; + return { + role: message?.role, + content: message?.content, + finish: choice?.finish_reason, + }; + } + return { + role: observed?.role, + content: observed?.content, + finish: observed?.stop_reason, + }; +} + +/** Project a raw chat response onto the canonical structural shape. */ +function drillNormalizeResponse(observed: unknown): { + role: unknown; + contentNonEmpty: boolean; + finishReasonInEnum: boolean; +} { + const { role, content, finish } = drillExtractChatStructure(observed); + return { + role, + contentNonEmpty: typeof content === "string" && content.trim().length > 0, + finishReasonInEnum: DRILL_FINISH_REASONS.has(finish as string), + }; +} + +/** + * Live plane is enabled iff a base URL AND an API key resolved from env. + * `$env` refs resolve to "" when unset, so this is false in replay-only + * (no-creds) runs and true only when the vector's endpoint+apiKey env vars are + * both populated. + */ +function drillLiveEnabled(input: any): boolean { + return Boolean(input.endpoint && input.apiKey); +} + +/** + * Run the drill: bind transport, replay the cassette, assert 3 planes. Returns + * the raw replay response; the runner then applies `normalize` and compares + * against the vector `expected` (the semantic plane). + */ +async function drillInvoke( + input: any, + context: AdapterContext, +): Promise { + const vector = (context.vector ?? {}) as Record; + const exchangeRaw = (vector.exchange ?? {}) as Record; + // The runner resolves refs only on `input`; the adapter owns `exchange` + // resolution (nested `$env` transport ref + `$json` cassette ref). + const exchange = context.resolveInput(exchangeRaw) as Record; + + const cassette = (exchange.cassette ?? {}) as Record; + const cassetteRequest = (cassette.request ?? {}) as Record; + const cassetteResponse = (cassette.response ?? {}) as Record; + const recordedRequest = cassetteRequest.body; + const recordedResponse = cassetteResponse.body; + const recordedStatus = Number(cassetteResponse.status ?? 200); + if (recordedResponse === undefined) { + throw new Error("drill cassette missing response.body"); + } + + // --- Replay: bind the transport to the local cassette server ------------- + const server = new CassetteReplayServer(recordedResponse, recordedStatus); + let raw: unknown; + await server.start(); + try { + raw = await drillExecute(input, { + endpoint: server.baseUrl, + apiKey: DRILL_REPLAY_API_KEY, + }); + + // Plane 1: transport -- the seam was reached and a request was captured. + const captured = server.lastRequest; + if (captured === null) { + throw new Error( + "drill transport plane: no request reached the replay server", + ); + } + if (raw === null || raw === undefined) { + throw new Error("drill transport plane: executor returned no response"); + } + + // Plane 2: wire -- outbound request body matches the cassette request. + if (recordedRequest !== undefined) { + const observedBody = captured.body; + if (drillCanonical(observedBody) !== drillCanonical(recordedRequest)) { + throw new Error( + "drill wire plane mismatch:\n" + + ` observed=${drillCanonical(observedBody)}\n` + + ` cassette=${drillCanonical(recordedRequest)}`, + ); + } + } + } finally { + await server.stop(); + } + + // --- Live parity (self-gated): live-semantic must equal replay-semantic --- + if (drillLiveEnabled(input)) { + const liveRaw = await drillExecute(input, { + endpoint: input.endpoint || null, + apiKey: input.apiKey, + }); + if ( + drillCanonical(drillNormalizeResponse(liveRaw)) !== + drillCanonical(drillNormalizeResponse(raw)) + ) { + throw new Error( + "drill live/replay parity mismatch:\n" + + ` live=${drillCanonical(drillNormalizeResponse(liveRaw))}\n` + + ` replay=${drillCanonical(drillNormalizeResponse(raw))}`, + ); + } + } + + // Plane 3 (semantic) is asserted by the runner: normalize(raw) == expected. + return raw; +} + +/** Registered semantic projection: raw response -> canonical structure. */ +function drillNormalize(observed: unknown, _context: AdapterContext): unknown { + return drillNormalizeResponse(observed); +} + // --------------------------------------------------------------------------- // Adapter registry // --------------------------------------------------------------------------- @@ -1076,6 +1332,10 @@ export const vectorAdapters = { "TurnConformance.replay": { invoke: replayInvoke }, "DiscoveryConformance.enrich": { invoke: discoveryEnrichInvoke }, "DiscoveryConformance.mapModel": { invoke: discoveryMapInvoke }, + "LiveChatConformance.complete": { + invoke: drillInvoke, + normalize: drillNormalize, + }, }; // Every cross-runtime contract now has a real TypeScript adapter above; there diff --git a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts index 11ba18e58..95027d62d 100644 --- a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts +++ b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts @@ -130,6 +130,12 @@ describe("callable vector conformance", () => { ) as Record; await runVector("DiscoveryConformance", "mapModel", vector, false, seam); }); + it("LiveChatConformance.complete:openai_chat_drill", async () => { + const vector = JSON.parse( + '{"name":"openai_chat_drill","stage":"drill","provider":"openai","input":{"provider":"openai","model":"gpt-4o-mini","apiKey":{"$env":"OPENAI_API_KEY"},"endpoint":{"$env":"OPENAI_BASE_URL"},"messages":[{"role":"user","content":"Say hello in exactly one word."}],"options":{"temperature":0,"maxOutputTokens":16}},"exchange":{"transport":{"baseUrl":{"$env":"OPENAI_BASE_URL"}},"cassette":{"$json":"cassettes/openai_chat_drill.json"},"planes":["transport","wire","semantic"]},"expected":{"role":"assistant","contentNonEmpty":true,"finishReasonInEnum":true},"operation":"complete"}', + ) as Record; + await runVector("LiveChatConformance", "complete", vector, false, seam); + }); it("LoadConformance.load:basic_load", async () => { const vector = JSON.parse( '{"name":"basic_load","description":"Load basic.prompty with env var resolution and verify all PromptAgent fields","stage":"load","input":{"fixture":"basic.prompty","env":{"OPENAI_ENDPOINT":"https://test.openai.com","OPENAI_API_KEY":"sk-test123"}},"expected":{"kind":"prompt","name":"basic-prompt","description":"A basic prompt for testing","metadata":{"authors":["testauthor"]},"model":{"id":"gpt-4","provider":"openai","apiType":"chat","connection":{"kind":"key","endpoint":"https://test.openai.com"},"options":{"temperature":0.7,"maxOutputTokens":1000}},"inputs":[{"name":"firstName","kind":"string","default":"Jane"},{"name":"lastName","kind":"string","default":"Doe"},{"name":"question","kind":"string","default":"What is the meaning of life?"}],"template":{"format":{"kind":"jinja2"},"parser":{"kind":"prompty"}},"instructions":"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}"},"operation":"load"}', diff --git a/schema/model/conformance/seams/live-chat-conformance.tsp b/schema/model/conformance/seams/live-chat-conformance.tsp new file mode 100644 index 000000000..cb3581e92 --- /dev/null +++ b/schema/model/conformance/seams/live-chat-conformance.tsp @@ -0,0 +1,24 @@ +import "@typra/emitter"; + +using Prompty.Conformance.Vectors; +namespace Prompty.Conformance.Seams; + +/** + * Conformance-only seam hosting the "service drill" vectors: a conformance + * vector that crosses the REAL Executor->Processor transport seam instead of + * stopping at the SDK edge. A drill runs against a live provider when + * credentials are present and REPLAYS a recorded cassette when they are not; + * the live<->replay parity is the gate. + * + * Like every other conformance seam this interface exists purely to carry + * @vector evidence into the emitted cross-runtime vectors.json; no runtime + * implements it directly. The consumer-owned drill adapter (roster) reads the + * vector's `exchange` block -- an UNKNOWN key that rides through the emitter + * untouched -- to bind the transport and assert the transport/wire/semantic + * planes. Nothing in Typra needs to know the word "drill". + */ +interface LiveChatConformance { + @doc("Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes") + @vector(DrillVectors) + complete(input: unknown): unknown; +} diff --git a/schema/model/conformance/vectors/drill.tsp b/schema/model/conformance/vectors/drill.tsp new file mode 100644 index 000000000..467804478 --- /dev/null +++ b/schema/model/conformance/vectors/drill.tsp @@ -0,0 +1,55 @@ +import "@typra/emitter"; + +namespace Prompty.Conformance.Vectors; + +// The first prompty "service drill" -- a conformance vector that crosses the +// REAL Executor->Processor transport seam. Bound to LiveChatConformance.complete. +// +// Authored as a JSON-STRING vector set (Typra 0.7.0+ affordance) so the domain +// request may carry keyword keys (e.g. "model") and nested ref objects. Two keys +// are intentionally NON-STANDARD and prove the emitter's unknown-key passthrough: +// +// * "exchange" -- the drill's transport/cassette/plane contract. The runner +// does NOT resolve it (it only resolves "input"); the consumer-owned adapter +// reads context.vector["exchange"] and resolves the nested $env base-url ref +// and $json cassette ref ITSELF via context.resolveInput. +// +// "requires" is deliberately OMITTED. The runner's requirement guard is +// all-or-nothing (an unavailable token skips the WHOLE vector), which cannot +// express "defer the live plane but still assert replay". So the drill vector +// carries no requirement token; the adapter self-gates the live plane on +// base-url + api-key presence and ALWAYS asserts the replay plane. +const DrillVectors = """ + [ + { + "name": "openai_chat_drill", + "stage": "drill", + "provider": "openai", + "input": { + "provider": "openai", + "model": "gpt-4o-mini", + "apiKey": { "$env": "OPENAI_API_KEY" }, + "endpoint": { "$env": "OPENAI_BASE_URL" }, + "messages": [ + { "role": "user", "content": "Say hello in exactly one word." } + ], + "options": { + "temperature": 0, + "maxOutputTokens": 16 + } + }, + "exchange": { + "transport": { + "baseUrl": { "$env": "OPENAI_BASE_URL" } + }, + "cassette": { "$json": "cassettes/openai_chat_drill.json" }, + "planes": ["transport", "wire", "semantic"] + }, + "expected": { + "role": "assistant", + "contentNonEmpty": true, + "finishReasonInEnum": true + } + } + ] + """; diff --git a/schema/model/main.tsp b/schema/model/main.tsp index a74a98768..4a865ac23 100644 --- a/schema/model/main.tsp +++ b/schema/model/main.tsp @@ -82,6 +82,10 @@ import "./conformance/seams/load-conformance.tsp"; import "./conformance/seams/wire-conformance.tsp"; import "./conformance/seams/discovery-conformance.tsp"; import "./conformance/seams/turn-conformance.tsp"; +// Service-drill seam: carries the transport-crossing drill vector. Emitter-change +// free by design -- the drill's non-standard "exchange" key rides through as an +// unknown vector key and is consumed only by the runtime's roster adapter. +import "./conformance/seams/live-chat-conformance.tsp"; // Behavior vectors (conformance evidence attached to callable operations) import "./conformance/vectors/render.tsp"; @@ -106,6 +110,10 @@ import "./conformance/vectors/replay.tsp"; // spec/vectors/process/stream_failure_vectors.json into the typra vector system so // every runtime is held to the same streaming-failure semantics. import "./conformance/vectors/stream.tsp"; +// Service-drill vector (LiveChatConformance.complete): the first vector to cross +// the real Executor->Processor transport seam. Carries a non-standard "exchange" +// block that the emitter passes through untouched into every runtime's harness. +import "./conformance/vectors/drill.tsp"; // Events import "./contracts/events/payloads.tsp"; diff --git a/schema/tsp-output/.typra-generated/export-surfaces.json b/schema/tsp-output/.typra-generated/export-surfaces.json index bf06d676b..58515d065 100644 --- a/schema/tsp-output/.typra-generated/export-surfaces.json +++ b/schema/tsp-output/.typra-generated/export-surfaces.json @@ -106,6 +106,7 @@ "InvocationUsage", "InvokerError", "Jinja2Format", + "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -288,6 +289,13 @@ "source": "HostToolExecutor.cs", "protocol": true }, + { + "name": "LiveChatConformance", + "kind": "type", + "group": "", + "source": "LiveChatConformance.cs", + "protocol": true + }, { "name": "LoadConformance", "kind": "type", @@ -2093,6 +2101,26 @@ } ] }, + { + "name": "LiveChatConformance", + "group": "", + "symbol": "LiveChatConformance", + "source": "LiveChatConformance.cs", + "methods": [ + { + "name": "complete", + "returns": "unknown", + "params": { + "input": "unknown" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": false, + "nonFatal": false + } + ] + }, { "name": "LoadConformance", "group": "", @@ -2433,6 +2461,7 @@ "EventSink.cs", "Executor.cs", "HostToolExecutor.cs", + "LiveChatConformance.cs", "LoadConformance.cs", "ModelLister.cs", "Parser.cs", @@ -2645,6 +2674,7 @@ "InvocationUsage", "InvokerError", "Jinja2Format", + "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -2827,6 +2857,13 @@ "source": "host_tool_executor.go", "protocol": true }, + { + "name": "LiveChatConformance", + "kind": "type", + "group": "", + "source": "live_chat_conformance.go", + "protocol": true + }, { "name": "LoadConformance", "kind": "type", @@ -4632,6 +4669,26 @@ } ] }, + { + "name": "LiveChatConformance", + "group": "", + "symbol": "LiveChatConformance", + "source": "live_chat_conformance.go", + "methods": [ + { + "name": "complete", + "returns": "unknown", + "params": { + "input": "unknown" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": false, + "nonFatal": false + } + ] + }, { "name": "LoadConformance", "group": "", @@ -5017,6 +5074,7 @@ "invocation_context_state.go", "invocation_usage.go", "invoker_error.go", + "live_chat_conformance.go", "llm_complete_payload.go", "llm_start_payload.go", "load_conformance.go", @@ -5184,6 +5242,7 @@ "InvocationUsage", "InvokerError", "Jinja2Format", + "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -5366,6 +5425,13 @@ "source": "HostToolExecutor.java", "protocol": true }, + { + "name": "LiveChatConformance", + "kind": "type", + "group": "", + "source": "LiveChatConformance.java", + "protocol": true + }, { "name": "LoadConformance", "kind": "type", @@ -7171,6 +7237,26 @@ } ] }, + { + "name": "LiveChatConformance", + "group": "", + "symbol": "LiveChatConformance", + "source": "LiveChatConformance.java", + "methods": [ + { + "name": "complete", + "returns": "unknown", + "params": { + "input": "unknown" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": false, + "nonFatal": false + } + ] + }, { "name": "LoadConformance", "group": "", @@ -7556,6 +7642,7 @@ "InvocationContextState.java", "InvocationUsage.java", "InvokerError.java", + "LiveChatConformance.java", "LlmCompletePayload.java", "LlmStartPayload.java", "LoadConformance.java", @@ -7722,6 +7809,7 @@ "InvocationUsage", "InvokerError", "Jinja2Format", + "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -7904,6 +7992,13 @@ "source": "HostToolExecutor", "protocol": true }, + { + "name": "LiveChatConformance", + "kind": "type", + "group": "", + "source": "LiveChatConformance", + "protocol": true + }, { "name": "LoadConformance", "kind": "type", @@ -9709,6 +9804,26 @@ } ] }, + { + "name": "LiveChatConformance", + "group": "", + "symbol": "LiveChatConformance", + "source": "LiveChatConformance", + "methods": [ + { + "name": "complete", + "returns": "unknown", + "params": { + "input": "unknown" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": false, + "nonFatal": false + } + ] + }, { "name": "LoadConformance", "group": "", @@ -10094,6 +10209,7 @@ "InvocationContextState", "InvocationUsage", "InvokerError", + "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -10261,6 +10377,7 @@ "InvocationUsage", "InvokerError", "Jinja2Format", + "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -10443,6 +10560,13 @@ "source": "._HostToolExecutor", "protocol": true }, + { + "name": "LiveChatConformance", + "kind": "type", + "group": "", + "source": "._LiveChatConformance", + "protocol": true + }, { "name": "LoadConformance", "kind": "type", @@ -12248,6 +12372,26 @@ } ] }, + { + "name": "LiveChatConformance", + "group": "", + "symbol": "LiveChatConformance", + "source": "._LiveChatConformance", + "methods": [ + { + "name": "complete", + "returns": "unknown", + "params": { + "input": "unknown" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": false, + "nonFatal": false + } + ] + }, { "name": "LoadConformance", "group": "", @@ -12588,6 +12732,7 @@ "._EventSink", "._Executor", "._HostToolExecutor", + "._LiveChatConformance", "._LoadConformance", "._ModelLister", "._Parser", @@ -12677,6 +12822,7 @@ "InvocationUsage", "InvokerError", "Jinja2Format", + "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -12859,6 +13005,13 @@ "source": "host_tool_executor", "protocol": true }, + { + "name": "LiveChatConformance", + "kind": "type", + "group": "", + "source": "live_chat_conformance", + "protocol": true + }, { "name": "LoadConformance", "kind": "type", @@ -14664,6 +14817,26 @@ } ] }, + { + "name": "LiveChatConformance", + "group": "", + "symbol": "LiveChatConformance", + "source": "live_chat_conformance", + "methods": [ + { + "name": "complete", + "returns": "unknown", + "params": { + "input": "unknown" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": false, + "nonFatal": false + } + ] + }, { "name": "LoadConformance", "group": "", @@ -15006,6 +15179,7 @@ "event_sink", "executor", "host_tool_executor", + "live_chat_conformance", "load_conformance", "model_lister", "operations", @@ -15095,6 +15269,7 @@ "InvocationUsage", "InvokerError", "Jinja2Format", + "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -15277,6 +15452,13 @@ "source": "Sources/PromptyModel/host_tool_executor.swift", "protocol": true }, + { + "name": "LiveChatConformance", + "kind": "type", + "group": "", + "source": "Sources/PromptyModel/live_chat_conformance.swift", + "protocol": true + }, { "name": "LoadConformance", "kind": "type", @@ -17082,6 +17264,26 @@ } ] }, + { + "name": "LiveChatConformance", + "group": "", + "symbol": "LiveChatConformance", + "source": "Sources/PromptyModel/live_chat_conformance.swift", + "methods": [ + { + "name": "complete", + "returns": "unknown", + "params": { + "input": "unknown" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": false, + "nonFatal": false + } + ] + }, { "name": "LoadConformance", "group": "", @@ -17536,6 +17738,7 @@ "Sources/PromptyModel/event_sink.swift", "Sources/PromptyModel/executor.swift", "Sources/PromptyModel/host_tool_executor.swift", + "Sources/PromptyModel/live_chat_conformance.swift", "Sources/PromptyModel/load_conformance.swift", "Sources/PromptyModel/model_lister.swift", "Sources/PromptyModel/operations/render_segment.swift", @@ -17634,6 +17837,7 @@ "InvocationUsage", "InvokerError", "Jinja2Format", + "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -17816,6 +18020,13 @@ "source": "./host-tool-executor", "protocol": true }, + { + "name": "LiveChatConformance", + "kind": "type", + "group": "", + "source": "./live-chat-conformance", + "protocol": true + }, { "name": "LoadConformance", "kind": "type", @@ -19621,6 +19832,26 @@ } ] }, + { + "name": "LiveChatConformance", + "group": "", + "symbol": "LiveChatConformance", + "source": "./live-chat-conformance", + "methods": [ + { + "name": "complete", + "returns": "unknown", + "params": { + "input": "unknown" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": false, + "nonFatal": false + } + ] + }, { "name": "LoadConformance", "group": "", @@ -20075,6 +20306,7 @@ "./event-sink", "./executor", "./host-tool-executor", + "./live-chat-conformance", "./load-conformance", "./model-lister", "./operations/render-segment", diff --git a/schema/tsp-output/.typra-generated/hydration-seams.json b/schema/tsp-output/.typra-generated/hydration-seams.json index b9d874328..3075ffb6b 100644 --- a/schema/tsp-output/.typra-generated/hydration-seams.json +++ b/schema/tsp-output/.typra-generated/hydration-seams.json @@ -84,6 +84,14 @@ "generatedSource": "HostToolExecutor.cs", "seamKind": "protocol-adapter" }, + { + "contract": "LiveChatConformance", + "target": "csharp", + "group": "", + "symbol": "LiveChatConformance", + "generatedSource": "LiveChatConformance.cs", + "seamKind": "protocol-adapter" + }, { "contract": "LoadConformance", "target": "csharp", @@ -228,6 +236,14 @@ "generatedSource": "host_tool_executor.go", "seamKind": "protocol-adapter" }, + { + "contract": "LiveChatConformance", + "target": "go", + "group": "", + "symbol": "LiveChatConformance", + "generatedSource": "live_chat_conformance.go", + "seamKind": "protocol-adapter" + }, { "contract": "LoadConformance", "target": "go", @@ -372,6 +388,14 @@ "generatedSource": "HostToolExecutor.java", "seamKind": "protocol-adapter" }, + { + "contract": "LiveChatConformance", + "target": "java", + "group": "", + "symbol": "LiveChatConformance", + "generatedSource": "LiveChatConformance.java", + "seamKind": "protocol-adapter" + }, { "contract": "LoadConformance", "target": "java", @@ -516,6 +540,14 @@ "generatedSource": "HostToolExecutor", "seamKind": "protocol-adapter" }, + { + "contract": "LiveChatConformance", + "target": "markdown", + "group": "", + "symbol": "LiveChatConformance", + "generatedSource": "LiveChatConformance", + "seamKind": "protocol-adapter" + }, { "contract": "LoadConformance", "target": "markdown", @@ -660,6 +692,14 @@ "generatedSource": "._HostToolExecutor", "seamKind": "protocol-adapter" }, + { + "contract": "LiveChatConformance", + "target": "python", + "group": "", + "symbol": "LiveChatConformance", + "generatedSource": "._LiveChatConformance", + "seamKind": "protocol-adapter" + }, { "contract": "LoadConformance", "target": "python", @@ -804,6 +844,14 @@ "generatedSource": "host_tool_executor", "seamKind": "protocol-adapter" }, + { + "contract": "LiveChatConformance", + "target": "rust", + "group": "", + "symbol": "LiveChatConformance", + "generatedSource": "live_chat_conformance", + "seamKind": "protocol-adapter" + }, { "contract": "LoadConformance", "target": "rust", @@ -948,6 +996,14 @@ "generatedSource": "Sources/PromptyModel/host_tool_executor.swift", "seamKind": "protocol-adapter" }, + { + "contract": "LiveChatConformance", + "target": "swift", + "group": "", + "symbol": "LiveChatConformance", + "generatedSource": "Sources/PromptyModel/live_chat_conformance.swift", + "seamKind": "protocol-adapter" + }, { "contract": "LoadConformance", "target": "swift", @@ -1092,6 +1148,14 @@ "generatedSource": "./host-tool-executor", "seamKind": "protocol-adapter" }, + { + "contract": "LiveChatConformance", + "target": "typescript", + "group": "", + "symbol": "LiveChatConformance", + "generatedSource": "./live-chat-conformance", + "seamKind": "protocol-adapter" + }, { "contract": "LoadConformance", "target": "typescript", diff --git a/schema/tsp-output/.typra-generated/manifest.json b/schema/tsp-output/.typra-generated/manifest.json index ff712eb29..f58c5dbb1 100644 --- a/schema/tsp-output/.typra-generated/manifest.json +++ b/schema/tsp-output/.typra-generated/manifest.json @@ -1333,6 +1333,11 @@ "path": "../runtime/csharp/Prompty.Core/Model/HostToolExecutor.cs", "marker": true }, + { + "outputRoot": "../runtime/csharp/Prompty.Core/Model", + "path": "../runtime/csharp/Prompty.Core/Model/LiveChatConformance.cs", + "marker": true + }, { "outputRoot": "../runtime/csharp/Prompty.Core/Model", "path": "../runtime/csharp/Prompty.Core/Model/LoadConformance.cs", @@ -1903,6 +1908,11 @@ "path": "../runtime/go/prompty/model/jinja2format_test.go", "marker": true }, + { + "outputRoot": "../runtime/go/prompty/model", + "path": "../runtime/go/prompty/model/live_chat_conformance.go", + "marker": true + }, { "outputRoot": "../runtime/go/prompty/model", "path": "../runtime/go/prompty/model/llm_complete_payload.go", @@ -3033,6 +3043,11 @@ "path": "../runtime/java/prompty/src/main/java/com/microsoft/prompty/model/Jinja2Format.java", "marker": true }, + { + "outputRoot": "../runtime/java/prompty/src/main/java/com/microsoft/prompty/model", + "path": "../runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LiveChatConformance.java", + "marker": true + }, { "outputRoot": "../runtime/java/prompty/src/main/java/com/microsoft/prompty/model", "path": "../runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LlmCompletePayload.java", @@ -4198,6 +4213,11 @@ "path": "../runtime/python/prompty/prompty/model/_HostToolExecutor.py", "marker": true }, + { + "outputRoot": "../runtime/python/prompty/prompty/model", + "path": "../runtime/python/prompty/prompty/model/_LiveChatConformance.py", + "marker": true + }, { "outputRoot": "../runtime/python/prompty/prompty/model", "path": "../runtime/python/prompty/prompty/model/_LoadConformance.py", @@ -6103,6 +6123,11 @@ "path": "../runtime/rust/prompty/src/model/host_tool_executor.rs", "marker": true }, + { + "outputRoot": "../runtime/rust/prompty/src/model", + "path": "../runtime/rust/prompty/src/model/live_chat_conformance.rs", + "marker": true + }, { "outputRoot": "../runtime/rust/prompty/src/model", "path": "../runtime/rust/prompty/src/model/load_conformance.rs", @@ -7213,6 +7238,11 @@ "path": "../runtime/swift/prompty-model/Sources/PromptyModel/host_tool_executor.swift", "marker": true }, + { + "outputRoot": "../runtime/swift/prompty-model", + "path": "../runtime/swift/prompty-model/Sources/PromptyModel/live_chat_conformance.swift", + "marker": true + }, { "outputRoot": "../runtime/swift/prompty-model", "path": "../runtime/swift/prompty-model/Sources/PromptyModel/load_conformance.swift", @@ -8313,6 +8343,11 @@ "path": "../runtime/typescript/packages/core/src/model/index.ts", "marker": true }, + { + "outputRoot": "../runtime/typescript/packages/core/src/model", + "path": "../runtime/typescript/packages/core/src/model/live-chat-conformance.ts", + "marker": true + }, { "outputRoot": "../runtime/typescript/packages/core/src/model", "path": "../runtime/typescript/packages/core/src/model/load-conformance.ts", @@ -9233,6 +9268,11 @@ "path": "../web/src/content/docs/reference/Jinja2Format.md", "marker": true }, + { + "outputRoot": "../web/src/content/docs/reference", + "path": "../web/src/content/docs/reference/LiveChatConformance.md", + "marker": true + }, { "outputRoot": "../web/src/content/docs/reference", "path": "../web/src/content/docs/reference/LlmCompletePayload.md", diff --git a/schema/tsp-output/.typra-generated/vectors.json b/schema/tsp-output/.typra-generated/vectors.json index f9a332c06..7da1d9af3 100644 --- a/schema/tsp-output/.typra-generated/vectors.json +++ b/schema/tsp-output/.typra-generated/vectors.json @@ -567,6 +567,63 @@ "operation": "mapModel" } }, + { + "contract": "LiveChatConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "complete", + "params": { + "input": "unknown" + }, + "returns": "unknown", + "sync": false, + "vector": { + "name": "openai_chat_drill", + "stage": "drill", + "provider": "openai", + "input": { + "provider": "openai", + "model": "gpt-4o-mini", + "apiKey": { + "$env": "OPENAI_API_KEY" + }, + "endpoint": { + "$env": "OPENAI_BASE_URL" + }, + "messages": [ + { + "role": "user", + "content": "Say hello in exactly one word." + } + ], + "options": { + "temperature": 0, + "maxOutputTokens": 16 + } + }, + "exchange": { + "transport": { + "baseUrl": { + "$env": "OPENAI_BASE_URL" + } + }, + "cassette": { + "$json": "cassettes/openai_chat_drill.json" + }, + "planes": [ + "transport", + "wire", + "semantic" + ] + }, + "expected": { + "role": "assistant", + "contentNonEmpty": true, + "finishReasonInEnum": true + }, + "operation": "complete" + } + }, { "contract": "LoadConformance", "namespace": "Prompty.Conformance.Seams", diff --git a/web/src/content/docs/reference/LiveChatConformance.md b/web/src/content/docs/reference/LiveChatConformance.md new file mode 100644 index 000000000..e5a54d3df --- /dev/null +++ b/web/src/content/docs/reference/LiveChatConformance.md @@ -0,0 +1,40 @@ +--- +title: "LiveChatConformance" +description: "Documentation for the LiveChatConformance type." +slug: "reference/livechatconformance" +--- + + +Conformance-only seam hosting the "service drill" vectors: a conformance +vector that crosses the REAL Executor->Processor transport seam instead of +stopping at the SDK edge. A drill runs against a live provider when +credentials are present and REPLAYS a recorded cassette when they are not; +the live<->replay parity is the gate. + +Like every other conformance seam this interface exists purely to carry + +## Class Diagram + +```mermaid +--- +title: LiveChatConformance +config: + look: handDrawn + theme: colorful + class: + hideEmptyMembersBox: true +--- +classDiagram + class LiveChatConformance { + <> + +complete(input: unknown) unknown [async-capable] + } +``` + +## Helper Methods + +The following helper methods are declared via `@method` and must be implemented by every runtime. The schema declares the logical protocol contract; each runtime maps async-capable methods to idiomatic sync/async shapes for that language. + +| Name | Signature | Runtime shape | Description | +| ---- | --------- | ------------- | ----------- | +| `complete` | `complete(input: unknown) -> unknown` | async-capable | Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes | From b22ffafc69461cf7a4ce87b2fe137cc55a42b331 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Fri, 4 Sep 2026 13:54:13 -0700 Subject: [PATCH 2/3] test(python): mock-fallback integration harness for service tests Service integration tests previously skipped entirely when API-key env vars were absent, leaving the executor->processor pipeline uncovered in CI. Add a mock-fallback harness so the same test bodies run against real services when creds are present and against deterministic mocks when they are not, driving coverage of the full provider pipeline in both modes with zero production-code changes. - Add tests/integration/mock_clients.py: request-aware sync+async mock clients returning real OpenAI SDK types (ChatCompletion, chunk streams, embeddings, images) and Anthropic SDK-shaped objects (attribute-access message + content blocks), covering chat, tool-call loops, structured output, streaming, embeddings and image generation. - conftest.py: autouse fixture patches only absent providers SDK constructors (openai.OpenAI/AsyncOpenAI/AzureOpenAI/AsyncAzureOpenAI, anthropic.Anthropic/AsyncAnthropic) and sets placeholder creds, so live providers are untouched (mixed real/mock mode works). Adds run_* markers. - Convert value-path test files (chat, streaming, structured, agent, embedding, image, anthropic) from skip_* to run_* decorators. Live-only tests (entra, model listing, tracing, direct) intentionally left skipped. Verified: 30 passed / 26 skipped in pure mock mode; 1371 unit tests green; ruff clean. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: faf27021-e24a-410d-bd40-fd03f22236e8 (cherry picked from commit 945f31a11f63a080401d85019b09089f90733ad1) --- .../prompty/tests/integration/conftest.py | 79 ++- .../prompty/tests/integration/mock_clients.py | 482 ++++++++++++++++++ .../prompty/tests/integration/test_agent.py | 6 +- .../tests/integration/test_anthropic.py | 12 +- .../prompty/tests/integration/test_chat.py | 6 +- .../tests/integration/test_embedding.py | 8 +- .../prompty/tests/integration/test_image.py | 8 +- .../tests/integration/test_streaming.py | 6 +- .../tests/integration/test_structured.py | 6 +- 9 files changed, 583 insertions(+), 30 deletions(-) create mode 100644 runtime/python/prompty/tests/integration/mock_clients.py diff --git a/runtime/python/prompty/tests/integration/conftest.py b/runtime/python/prompty/tests/integration/conftest.py index 40dc2de22..5eda22bf2 100644 --- a/runtime/python/prompty/tests/integration/conftest.py +++ b/runtime/python/prompty/tests/integration/conftest.py @@ -87,6 +87,77 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: reason="AZURE_OPENAI_ENDPOINT or AZURE_OPENAI_CHAT_DEPLOYMENT not set (Entra ID tests)", ) +# --------------------------------------------------------------------------- +# Mock-fallback markers +# --------------------------------------------------------------------------- +# +# Value-path tests (chat / embedding / image / streaming / structured / agent) +# always run: with real credentials they hit the live service, without them an +# autouse fixture (below) patches the SDK constructors so the same test bodies +# drive the full executor -> processor pipeline against deterministic mocks. +# +# A ``run_*`` marker therefore never skips in mock mode. It only skips in *live* +# mode when a provider is configured but a specific sub-resource deployment +# (embedding / image) is not, since a real call needs that deployment name. + +_never_skip = pytest.mark.skipif(False, reason="") + +run_openai = _never_skip +run_openai_image = _never_skip +run_openai_embedding = _never_skip +run_foundry = _never_skip +run_foundry_embedding = pytest.mark.skipif( + has_foundry and not _AZURE_EMBEDDING_DEPLOYMENT, + reason="live Azure OpenAI configured but AZURE_OPENAI_EMBEDDING_DEPLOYMENT not set", +) +run_foundry_image = pytest.mark.skipif( + has_foundry and not _AZURE_IMAGE_DEPLOYMENT, + reason="live Azure OpenAI configured but AZURE_OPENAI_IMAGE_DEPLOYMENT not set", +) +run_anthropic = _never_skip + +# True when a provider will be served by mocks (its credentials are absent). +mock_openai = not has_openai +mock_foundry = not has_foundry +mock_anthropic = not has_anthropic + + +@pytest.fixture(autouse=True) +def _mock_absent_services(monkeypatch: pytest.MonkeyPatch) -> None: + """Install deterministic mock SDK clients for any provider lacking credentials. + + Executors import the SDK constructor lazily inside ``_resolve_client`` (e.g. + ``from openai import OpenAI``), so patching the module attribute here is picked + up at call time. Providers whose credentials *are* present are left untouched + and continue to exercise the real service. + """ + import openai + + from .mock_clients import MockAnthropicClient, MockOpenAIClient + + if mock_openai: + monkeypatch.setenv("OPENAI_API_KEY", _OPENAI_KEY or "mock-openai-key") + monkeypatch.setattr(openai, "OpenAI", lambda *a, **k: MockOpenAIClient(is_async=False, b64_images=False)) + monkeypatch.setattr(openai, "AsyncOpenAI", lambda *a, **k: MockOpenAIClient(is_async=True, b64_images=False)) + if mock_foundry: + monkeypatch.setenv("AZURE_OPENAI_API_KEY", _AZURE_KEY or "mock-azure-key") + monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", _AZURE_ENDPOINT or "https://mock.local/") + # Azure image generation returns base64 payloads; the foundry image test + # decodes them, so its mock must emit b64_json rather than a URL. + monkeypatch.setattr(openai, "AzureOpenAI", lambda *a, **k: MockOpenAIClient(is_async=False, b64_images=True)) + monkeypatch.setattr( + openai, "AsyncAzureOpenAI", lambda *a, **k: MockOpenAIClient(is_async=True, b64_images=True) + ) + if mock_anthropic: + monkeypatch.setenv("ANTHROPIC_API_KEY", _ANTHROPIC_KEY or "mock-anthropic-key") + try: + import anthropic + except ImportError: + anthropic = None + if anthropic is not None: + monkeypatch.setattr(anthropic, "Anthropic", lambda *a, **k: MockAnthropicClient(is_async=False)) + monkeypatch.setattr(anthropic, "AsyncAnthropic", lambda *a, **k: MockAnthropicClient(is_async=True)) + # --------------------------------------------------------------------------- # Agent helpers shared across test files @@ -115,7 +186,7 @@ def make_openai_agent( connection: dict[str, Any] = { "kind": "key", - "apiKey": _OPENAI_KEY, + "apiKey": _OPENAI_KEY or "mock-openai-key", } if _OPENAI_BASE_URL: connection["endpoint"] = _OPENAI_BASE_URL @@ -210,8 +281,8 @@ def make_foundry_agent( "apiType": api_type, "connection": { "kind": "key", - "endpoint": _AZURE_ENDPOINT, - "apiKey": _AZURE_KEY, + "endpoint": _AZURE_ENDPOINT or "https://mock.local/", + "apiKey": _AZURE_KEY or "mock-azure-key", }, }, } @@ -252,7 +323,7 @@ def make_anthropic_agent( "apiType": api_type, "connection": { "kind": "key", - "apiKey": _ANTHROPIC_KEY, + "apiKey": _ANTHROPIC_KEY or "mock-anthropic-key", }, }, } diff --git a/runtime/python/prompty/tests/integration/mock_clients.py b/runtime/python/prompty/tests/integration/mock_clients.py new file mode 100644 index 000000000..26010fade --- /dev/null +++ b/runtime/python/prompty/tests/integration/mock_clients.py @@ -0,0 +1,482 @@ +"""Deterministic, request-aware mock SDK clients for offline integration runs. + +The integration suite runs against real services when credentials are present and +against these mocks when they are not (see ``conftest.py``). The goal is coverage, +not fidelity: the same test bodies drive the real executor wire-building and the +real processor extraction path in both modes. To do that faithfully the mocks +return genuine SDK type instances (``ChatCompletion``, ``CreateEmbeddingResponse``, +``ImagesResponse``, ``ChatCompletionChunk``) so the processor's ``isinstance`` +dispatch runs exactly as it does live. Anthropic responses are returned as plain +dicts, which its duck-typed processor accepts. + +Responses are shaped by the request so tool-call loops, structured output, +streaming, embeddings and image generation each exercise their real code path: + +* ``tools`` present and no prior tool result -> assistant message with a tool call +* a prior ``role: tool`` message present -> final answer echoing the tool output +* ``response_format`` present -> JSON matching the requested schema +* ``stream=True`` -> an iterator/async-iterator of chunks +""" + +from __future__ import annotations + +import base64 +import json +from collections.abc import AsyncIterator, Iterator +from typing import Any + +# --------------------------------------------------------------------------- +# SDK type builders (real instances so processor isinstance dispatch runs) +# --------------------------------------------------------------------------- + + +def _usage() -> Any: + from openai.types import CompletionUsage + + return CompletionUsage(prompt_tokens=11, completion_tokens=7, total_tokens=18) + + +def build_chat_completion( + *, + content: str | None = None, + tool_calls: list[dict[str, Any]] | None = None, + model: str = "mock-model", +) -> Any: + """Build a real ``ChatCompletion`` with a single choice.""" + from openai.types.chat import ChatCompletion, ChatCompletionMessage + from openai.types.chat.chat_completion import Choice + from openai.types.chat.chat_completion_message_tool_call import ( + ChatCompletionMessageToolCall, + Function, + ) + + sdk_tool_calls = None + finish_reason = "stop" + if tool_calls: + sdk_tool_calls = [ + ChatCompletionMessageToolCall( + id=tc["id"], + type="function", + function=Function(name=tc["name"], arguments=tc["arguments"]), + ) + for tc in tool_calls + ] + finish_reason = "tool_calls" + + message = ChatCompletionMessage( + role="assistant", + content=content, + tool_calls=sdk_tool_calls, + ) + choice = Choice(index=0, finish_reason=finish_reason, message=message) + return ChatCompletion( + id="chatcmpl-mock", + object="chat.completion", + created=0, + model=model, + choices=[choice], + usage=_usage(), + ) + + +def build_chunks(text: str, *, model: str = "mock-model") -> list[Any]: + """Split ``text`` into a sequence of real ``ChatCompletionChunk`` deltas.""" + from openai.types.chat import ChatCompletionChunk + from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice + from openai.types.chat.chat_completion_chunk import ChoiceDelta + + words = text.split(" ") + pieces = [(w + " " if i < len(words) - 1 else w) for i, w in enumerate(words)] + chunks: list[Any] = [] + for piece in pieces: + chunks.append( + ChatCompletionChunk( + id="chatcmpl-mock", + object="chat.completion.chunk", + created=0, + model=model, + choices=[ChunkChoice(index=0, delta=ChoiceDelta(content=piece), finish_reason=None)], + ) + ) + chunks.append( + ChatCompletionChunk( + id="chatcmpl-mock", + object="chat.completion.chunk", + created=0, + model=model, + choices=[ChunkChoice(index=0, delta=ChoiceDelta(), finish_reason="stop")], + ) + ) + return chunks + + +def build_embedding(inputs: Any, *, dims: int = 8) -> Any: + """Build a real ``CreateEmbeddingResponse`` with one vector per input.""" + from openai.types import CreateEmbeddingResponse, Embedding + from openai.types.create_embedding_response import Usage + + if isinstance(inputs, str): + count = 1 + elif isinstance(inputs, list): + count = max(1, len(inputs)) + else: + count = 1 + + data = [ + Embedding( + index=i, + object="embedding", + embedding=[round(0.1 * (i + j + 1), 4) for j in range(dims)], + ) + for i in range(count) + ] + return CreateEmbeddingResponse( + data=data, + model="mock-embedding", + object="list", + usage=Usage(prompt_tokens=4, total_tokens=4), + ) + + +def build_image(*, b64: bool = False) -> Any: + """Build a real ``ImagesResponse`` — URL by default, base64 when requested.""" + from openai.types import Image, ImagesResponse + + if b64: + payload = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"\x00" * 2048).decode("ascii") + image = Image(b64_json=payload, url=None) + else: + image = Image(url="https://mock.local/generated-image.png", b64_json=None) + return ImagesResponse(created=0, data=[image]) + + +# --------------------------------------------------------------------------- +# Request-aware response synthesis +# --------------------------------------------------------------------------- + + +def _synthesize_from_schema(schema: dict[str, Any]) -> dict[str, Any]: + props = schema.get("properties", {}) or {} + out: dict[str, Any] = {} + for name, spec in props.items(): + t = spec.get("type") + if isinstance(t, list): + t = next((x for x in t if x != "null"), "string") + if t == "integer": + out[name] = 123 + elif t == "number": + out[name] = 1.5 + elif t == "boolean": + out[name] = True + elif t == "array": + out[name] = [] + elif t == "object": + out[name] = {} + else: + out[name] = f"mock-{name}" + return out + + +def _structured_content(response_format: dict[str, Any]) -> str: + schema: dict[str, Any] = {} + if isinstance(response_format, dict): + js = response_format.get("json_schema") + if isinstance(js, dict): + schema = js.get("schema", {}) or {} + return json.dumps(_synthesize_from_schema(schema)) + + +def _first_tool_call(tools: list[dict[str, Any]]) -> dict[str, Any]: + tool = tools[0] + fn = tool.get("function", tool) + name = fn.get("name", "tool") + # OpenAI tools carry ``parameters``; Anthropic tools carry ``input_schema``. + schema = fn.get("parameters") or fn.get("input_schema") or {} + params = schema.get("properties", {}) or {} + args = {k: "Seattle" if v.get("type") in (None, "string") else 1 for k, v in params.items()} + return {"id": "call_mock_1", "name": name, "arguments": json.dumps(args)} + + +def _last_tool_output(messages: list[Any]) -> str | None: + for msg in reversed(messages): + if isinstance(msg, dict) and msg.get("role") == "tool": + content = msg.get("content") + return content if isinstance(content, str) else json.dumps(content) + return None + + +def _anthropic_tool_result(messages: list[Any]) -> str | None: + """Return the text of the most recent Anthropic ``tool_result`` block, if any.""" + for msg in reversed(messages): + if not isinstance(msg, dict): + continue + content = msg.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_result": + inner = block.get("content") + if isinstance(inner, str): + return inner + return json.dumps(inner) + return None + + +def _collect_user_text(messages: list[Any]) -> str: + """Concatenate the plain text of every user message (wire form).""" + parts: list[str] = [] + for msg in messages: + if not isinstance(msg, dict) or msg.get("role") != "user": + continue + content = msg.get("content") + if isinstance(content, str): + parts.append(content) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + parts.append(block.get("text", "")) + return " ".join(p for p in parts if p) + + +def chat_response_for(kwargs: dict[str, Any]) -> Any: + """Choose a ``ChatCompletion`` (or chunk stream) based on the request.""" + messages = kwargs.get("messages") or [] + tools = kwargs.get("tools") + response_format = kwargs.get("response_format") + streaming = bool(kwargs.get("stream")) + + if streaming: + return build_chunks("hello world") + + tool_output = _last_tool_output(messages) + if tool_output is not None: + return build_chat_completion(content=f"The weather result is: {tool_output}") + + if tools: + return build_chat_completion(tool_calls=[_first_tool_call(tools)]) + + if response_format: + return build_chat_completion(content=_structured_content(response_format)) + + return build_chat_completion(content="Hello! This is a mock assistant reply.") + + +def _anthropic_stream_events(text: str) -> list[dict[str, Any]]: + """Anthropic Messages streaming events whose text_deltas concatenate to ``text``.""" + events: list[dict[str, Any]] = [ + {"type": "message_start", "message": {"role": "assistant", "content": []}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ] + for word in text.split(" "): + events.append({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": word + " "}}) + events.append({"type": "content_block_stop", "index": 0}) + events.append({"type": "message_stop"}) + return events + + +def anthropic_response_for(kwargs: dict[str, Any]) -> Any: + """Anthropic Messages API response as a dict (its processor accepts dicts). + + When ``stream`` is set, returns a list of streaming event dicts instead. + """ + if kwargs.get("stream"): + return _anthropic_stream_events("hello world") + + messages = kwargs.get("messages") or [] + tools = kwargs.get("tools") + output_config = kwargs.get("output_config") + + tool_output = _anthropic_tool_result(messages) or _last_tool_output(messages) + if tool_output is not None: + blocks: list[dict[str, Any]] = [{"type": "text", "text": f"The weather result is: {tool_output}"}] + stop = "end_turn" + elif tools: + tc = _first_tool_call(tools) + blocks = [ + { + "type": "tool_use", + "id": tc["id"], + "name": tc["name"], + "input": json.loads(tc["arguments"]), + } + ] + stop = "tool_use" + elif output_config: + schema = {} + fmt = output_config.get("format") if isinstance(output_config, dict) else None + if isinstance(fmt, dict): + schema = fmt.get("schema", {}) or {} + blocks = [{"type": "text", "text": json.dumps(_synthesize_from_schema(schema))}] + stop = "end_turn" + else: + # Echo user context so multi-turn "remembers" earlier turns. + context = _collect_user_text(messages) + text = "Hello! This is a mock assistant reply." + if context: + text = f"{text} You said: {context}" + blocks = [{"type": "text", "text": text}] + stop = "end_turn" + + return { + "id": "msg_mock", + "type": "message", + "role": "assistant", + "model": kwargs.get("model", "mock-claude"), + "content": blocks, + "stop_reason": stop, + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 7}, + } + + +def _anthropic_message_object(payload: dict[str, Any]) -> Any: + """Wrap the Anthropic response dict as attribute-access objects. + + The real Anthropic SDK returns objects, and the pipeline agent loop inspects + responses via attribute access (``response.stop_reason``, ``block.type``), so + the mock must expose the same shape for the tool-call loop to iterate. + """ + from types import SimpleNamespace + + blocks = [] + for block in payload["content"]: + # ``input`` stays a plain dict, matching the SDK's tool_use blocks. + blocks.append(SimpleNamespace(**block)) + return SimpleNamespace( + id=payload["id"], + type=payload["type"], + role=payload["role"], + model=payload["model"], + content=blocks, + stop_reason=payload["stop_reason"], + stop_sequence=payload["stop_sequence"], + usage=SimpleNamespace(**payload["usage"]), + ) + + +# --------------------------------------------------------------------------- +# Iterators for streaming +# --------------------------------------------------------------------------- + + +class _SyncChunkStream(Iterator): + def __init__(self, chunks: list[Any]) -> None: + self._it = iter(chunks) + + def __iter__(self) -> _SyncChunkStream: + return self + + def __next__(self) -> Any: + return next(self._it) + + +class _AsyncChunkStream(AsyncIterator): + def __init__(self, chunks: list[Any]) -> None: + self._chunks = chunks + self._i = 0 + + def __aiter__(self) -> _AsyncChunkStream: + return self + + async def __anext__(self) -> Any: + if self._i >= len(self._chunks): + raise StopAsyncIteration + chunk = self._chunks[self._i] + self._i += 1 + return chunk + + +# --------------------------------------------------------------------------- +# Mock client objects mimicking the openai / anthropic client surface +# --------------------------------------------------------------------------- + + +class _ChatCompletions: + def __init__(self, is_async: bool) -> None: + self._is_async = is_async + + def create(self, **kwargs: Any) -> Any: + result = chat_response_for(kwargs) + if kwargs.get("stream"): + chunks = result if isinstance(result, list) else list(result) + if self._is_async: + return _wrap_async(_AsyncChunkStream(chunks)) + return _SyncChunkStream(chunks) + if self._is_async: + return _wrap_async(result) + return result + + +class _Embeddings: + def __init__(self, is_async: bool) -> None: + self._is_async = is_async + + def create(self, **kwargs: Any) -> Any: + result = build_embedding(kwargs.get("input")) + return _wrap_async(result) if self._is_async else result + + +class _Images: + def __init__(self, is_async: bool, *, b64: bool) -> None: + self._is_async = is_async + self._b64 = b64 + + def generate(self, **kwargs: Any) -> Any: + result = build_image(b64=self._b64) + return _wrap_async(result) if self._is_async else result + + +class _Chat: + def __init__(self, is_async: bool) -> None: + self.completions = _ChatCompletions(is_async) + + +class MockOpenAIClient: + """Mimics ``openai.OpenAI`` / ``openai.AzureOpenAI`` (and their async peers). + + ``b64_images`` controls whether image generation returns base64 (Azure-style) + or a URL (OpenAI-style), matching what each provider's tests expect. + """ + + def __init__(self, *args: Any, is_async: bool = False, b64_images: bool = False, **kwargs: Any) -> None: + self.chat = _Chat(is_async) + self.embeddings = _Embeddings(is_async) + self.images = _Images(is_async, b64=b64_images) + + +class _AnthropicMessages: + def __init__(self, is_async: bool) -> None: + self._is_async = is_async + + def create(self, **kwargs: Any) -> Any: + result = anthropic_response_for(kwargs) + if kwargs.get("stream"): + events = result if isinstance(result, list) else list(result) + if self._is_async: + return _wrap_async(_AsyncChunkStream(events)) + return _SyncChunkStream(events) + # Non-streaming: the pipeline agent loop inspects the response via + # attribute access, so wrap the dict as objects like the real SDK. + message = _anthropic_message_object(result) + return _wrap_async(message) if self._is_async else message + + +class MockAnthropicClient: + """Mimics ``anthropic.Anthropic`` / ``anthropic.AsyncAnthropic``.""" + + def __init__(self, *args: Any, is_async: bool = False, **kwargs: Any) -> None: + self.messages = _AnthropicMessages(is_async) + + +# --------------------------------------------------------------------------- +# Async helper — a coroutine yielding the given value +# --------------------------------------------------------------------------- + + +async def _identity(value: Any) -> Any: + return value + + +def _wrap_async(value: Any) -> Any: + """Return an awaitable that resolves to ``value`` (async ``create`` returns this).""" + return _identity(value) diff --git a/runtime/python/prompty/tests/integration/test_agent.py b/runtime/python/prompty/tests/integration/test_agent.py index 822d52f5e..82a585067 100644 --- a/runtime/python/prompty/tests/integration/test_agent.py +++ b/runtime/python/prompty/tests/integration/test_agent.py @@ -6,7 +6,7 @@ from prompty.core.pipeline import turn, turn_async -from .conftest import make_foundry_agent, make_openai_agent, skip_foundry, skip_openai +from .conftest import make_foundry_agent, make_openai_agent, run_foundry, run_openai _TOOLS = [ { @@ -34,7 +34,7 @@ def _weather_fn(city: str) -> str: # --------------------------------------------------------------------------- -@skip_openai +@run_openai class TestOpenAIAgent: def test_tool_call_loop(self): agent = make_openai_agent( @@ -70,7 +70,7 @@ async def test_async_tool_call_loop(self): # --------------------------------------------------------------------------- -@skip_foundry +@run_foundry class TestFoundryAgent: def test_tool_call_loop(self): agent = make_foundry_agent( diff --git a/runtime/python/prompty/tests/integration/test_anthropic.py b/runtime/python/prompty/tests/integration/test_anthropic.py index d755d9360..3880ce037 100644 --- a/runtime/python/prompty/tests/integration/test_anthropic.py +++ b/runtime/python/prompty/tests/integration/test_anthropic.py @@ -19,7 +19,7 @@ from prompty.providers.anthropic.executor import AnthropicExecutor from prompty.providers.anthropic.processor import AnthropicProcessor -from .conftest import make_anthropic_agent, skip_anthropic +from .conftest import make_anthropic_agent, run_anthropic executor = AnthropicExecutor() processor = AnthropicProcessor() @@ -32,7 +32,7 @@ def _hello_messages() -> list[Message]: ] -@skip_anthropic +@run_anthropic class TestAnthropicChat: def test_basic_chat(self): agent = make_anthropic_agent(options={"maxOutputTokens": 64}) @@ -61,7 +61,7 @@ def test_temperature_control(self): assert r1 == r2 -@skip_anthropic +@run_anthropic class TestAnthropicStructured: def test_structured_output(self): agent = make_anthropic_agent( @@ -83,7 +83,7 @@ def test_structured_output(self): assert "language" in result -@skip_anthropic +@run_anthropic class TestAnthropicAgent: def test_tool_calling(self): def get_weather(location: str) -> str: @@ -135,7 +135,7 @@ def test_full_agent_loop(self): assert len(result) > 0 -@skip_anthropic +@run_anthropic class TestAnthropicStreaming: def test_streaming_chat(self): agent = make_anthropic_agent( @@ -154,7 +154,7 @@ def test_streaming_chat(self): assert len(events) > 0 -@skip_anthropic +@run_anthropic class TestAnthropicMultiTurn: def test_multi_turn_conversation(self): """Test sending prior assistant/user turns — Anthropic is strict about role alternation.""" diff --git a/runtime/python/prompty/tests/integration/test_chat.py b/runtime/python/prompty/tests/integration/test_chat.py index bb46606aa..b021da332 100644 --- a/runtime/python/prompty/tests/integration/test_chat.py +++ b/runtime/python/prompty/tests/integration/test_chat.py @@ -10,7 +10,7 @@ from prompty.providers.openai.executor import OpenAIExecutor from prompty.providers.openai.processor import OpenAIProcessor -from .conftest import make_foundry_agent, make_openai_agent, skip_foundry, skip_openai +from .conftest import make_foundry_agent, make_openai_agent, run_foundry, run_openai def _hello_messages() -> list[Message]: @@ -28,7 +28,7 @@ def _hello_messages() -> list[Message]: # --------------------------------------------------------------------------- -@skip_openai +@run_openai class TestOpenAIChat: executor = OpenAIExecutor() processor = OpenAIProcessor() @@ -63,7 +63,7 @@ async def test_async_chat(self): # --------------------------------------------------------------------------- -@skip_foundry +@run_foundry class TestFoundryChat: executor = FoundryExecutor() processor = FoundryProcessor() diff --git a/runtime/python/prompty/tests/integration/test_embedding.py b/runtime/python/prompty/tests/integration/test_embedding.py index 12d154ac6..6ea53661a 100644 --- a/runtime/python/prompty/tests/integration/test_embedding.py +++ b/runtime/python/prompty/tests/integration/test_embedding.py @@ -14,8 +14,8 @@ _OPENAI_EMBEDDING_MODEL, make_foundry_agent, make_openai_agent, - skip_foundry_embedding, - skip_openai, + run_foundry_embedding, + run_openai_embedding, ) # --------------------------------------------------------------------------- @@ -23,7 +23,7 @@ # --------------------------------------------------------------------------- -@skip_openai +@run_openai_embedding class TestOpenAIEmbedding: executor = OpenAIExecutor() processor = OpenAIProcessor() @@ -58,7 +58,7 @@ async def test_async_embedding(self): # --------------------------------------------------------------------------- -@skip_foundry_embedding +@run_foundry_embedding class TestFoundryEmbedding: executor = FoundryExecutor() processor = FoundryProcessor() diff --git a/runtime/python/prompty/tests/integration/test_image.py b/runtime/python/prompty/tests/integration/test_image.py index 7a8524779..8ce5c3c94 100644 --- a/runtime/python/prompty/tests/integration/test_image.py +++ b/runtime/python/prompty/tests/integration/test_image.py @@ -14,12 +14,12 @@ _OPENAI_IMAGE_MODEL, make_foundry_agent, make_openai_agent, - skip_foundry_image, - skip_openai_image, + run_foundry_image, + run_openai_image, ) -@skip_openai_image +@run_openai_image class TestOpenAIImage: executor = OpenAIExecutor() processor = OpenAIProcessor() @@ -37,7 +37,7 @@ def test_image_generation(self): assert result.startswith("http") or len(base64.b64decode(result)) > 1000 -@skip_foundry_image +@run_foundry_image class TestFoundryImage: executor = FoundryExecutor() processor = FoundryProcessor() diff --git a/runtime/python/prompty/tests/integration/test_streaming.py b/runtime/python/prompty/tests/integration/test_streaming.py index 2d44b04b2..d9f409d4f 100644 --- a/runtime/python/prompty/tests/integration/test_streaming.py +++ b/runtime/python/prompty/tests/integration/test_streaming.py @@ -12,7 +12,7 @@ from prompty.providers.openai.executor import OpenAIExecutor from prompty.providers.openai.processor import OpenAIProcessor -from .conftest import make_foundry_agent, make_openai_agent, skip_foundry, skip_openai +from .conftest import make_foundry_agent, make_openai_agent, run_foundry, run_openai def _chat_messages() -> list[Message]: @@ -30,7 +30,7 @@ def _chat_messages() -> list[Message]: # --------------------------------------------------------------------------- -@skip_openai +@run_openai class TestOpenAIStreaming: executor = OpenAIExecutor() processor = OpenAIProcessor() @@ -85,7 +85,7 @@ async def test_async_streaming_chat(self): # --------------------------------------------------------------------------- -@skip_foundry +@run_foundry class TestFoundryStreaming: executor = FoundryExecutor() processor = FoundryProcessor() diff --git a/runtime/python/prompty/tests/integration/test_structured.py b/runtime/python/prompty/tests/integration/test_structured.py index aa3c72c32..3641dce86 100644 --- a/runtime/python/prompty/tests/integration/test_structured.py +++ b/runtime/python/prompty/tests/integration/test_structured.py @@ -10,7 +10,7 @@ from prompty.providers.openai.executor import OpenAIExecutor from prompty.providers.openai.processor import OpenAIProcessor -from .conftest import make_foundry_agent, make_openai_agent, skip_foundry, skip_openai +from .conftest import make_foundry_agent, make_openai_agent, run_foundry, run_openai _OUTPUT_SCHEMA = { "properties": [ @@ -43,7 +43,7 @@ def _structured_messages() -> list[Message]: # --------------------------------------------------------------------------- -@skip_openai +@run_openai class TestOpenAIStructuredOutput: executor = OpenAIExecutor() processor = OpenAIProcessor() @@ -82,7 +82,7 @@ async def test_async_structured_output(self): # --------------------------------------------------------------------------- -@skip_foundry +@run_foundry class TestFoundryStructuredOutput: executor = FoundryExecutor() processor = FoundryProcessor() From b150871237ec8a3784e0d769e8255e01b9afbf60 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Fri, 4 Sep 2026 16:25:33 -0700 Subject: [PATCH 3/3] revert(model): back service-drill vector out of the conformance rail Reverts d8cb188f. The service-drill spike answered its YAGNI-gate question, and the answer is "do not ship it in the emitted conformance rail." Why (the "no special flower" principle): every runtime's conformance surface must be identical and driven purely by Typra-emitted contracts -- no per-runtime extras that are not bound to a contract. The drill vector rides through the emitter fine (unknown stage:"drill" + exchange keys pass through unchanged), but everything that makes it *run* is irreducibly consumer-owned roster that the hard constraints forbid the emitter from ever owning: - the cassette-replay HTTP server - the SDK / base-URL transport binding - cassette bytes + exchange ref-resolution - the 3-plane (transport/wire/semantic) assertion - the var:live-enabled credential gate + live/replay parity Because the emitter can never generate that roster, authoring the drill as an emitted @vector forces EVERY runtime into one of two hand-authored, non-contract-bound choices: a drill adapter (Python, TS) or a waiver (Go, C#, Java, Rust, Swift). Both are "special flowers." There is no state in which the drill is simultaneously (a) an emitted uniform vector and (b) free of per-runtime roster. So it does not belong in the emitted conformance rail. This revert returns all 7 runtimes to the pre-drill #517 baseline (byte-identical for every schema, generated-model, runtime-model, and web file), so the conformance surface is once again uniform and contract-only. Verdict for the Typra follow-up: Typra does nothing. The drill is real x-N per-runtime duplication that the hard constraints keep out of the emitter, so there is no minimal stage:"drill"+exchange emitter rail worth scoping -- the emitter already passes unknown keys through, and the value-add would all be consumer roster. Service-quality probing, if wanted, lives in a separate consumer-owned integration layer (the mock-fallback harness in tests/integration/, kept on this branch), never the uniform conformance rail. Kept: the mock-fallback integration harness (b22ffafc) -- it is the consumer-owned, non-conformance integration layer where mixed real/mock service probing correctly belongs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 442651bd-34df-4b59-8e2f-d7093fa855a8 --- .../Model/VectorAdapters.cs | 5 +- .../Model/VectorConformanceTests.cs | 243 +++++------ .../Prompty.Core/Model/LiveChatConformance.cs | 31 -- .../go/prompty/model/live_chat_conformance.go | 17 - .../prompty/model/vector_conformance_test.go | 242 +++++------ .../go/prompty/model/vectorbridge/bridge.go | 21 - .../prompty/vectoradapters/vectoradapters.go | 4 +- .../prompty/model/LiveChatConformance.java | 12 - .../prompty/model/VectorAdapters.java | 6 +- .../prompty/model/VectorConformanceTests.java | 384 +++++++++--------- .../prompty/model/_LiveChatConformance.py | 28 -- .../python/prompty/prompty/model/__init__.py | 2 - .../prompty/tests/model/_drill_replay.py | 97 ----- .../model/cassettes/openai_chat_drill.json | 44 -- .../prompty/tests/model/record_cassette.py | 107 ----- .../model/test_capability_guard_reference.py | 17 +- .../tests/model/test_vector_conformance.py | 241 +++++------ .../prompty/tests/model/vector_adapters.py | 169 ++------ .../src/model/live_chat_conformance.rs | 21 - runtime/rust/prompty/src/model/mod.rs | 3 - .../prompty/tests/model/vector_adapters.rs | 5 +- .../tests/model/vector_conformance_test.rs | 195 +++++---- .../PromptyModel/live_chat_conformance.swift | 8 - .../PromptyModelTests/VectorAdapters.swift | 5 +- .../VectorConformanceTests.swift | 198 +++++---- .../packages/core/src/model/index.ts | 1 - .../core/src/model/live-chat-conformance.ts | 15 - .../model/cassettes/openai_chat_drill.json | 44 -- .../packages/core/tests/model/drill-replay.ts | 94 ----- .../core/tests/model/vector-adapters.ts | 260 ------------ .../tests/model/vector-conformance.test.ts | 6 - .../seams/live-chat-conformance.tsp | 24 -- schema/model/conformance/vectors/drill.tsp | 55 --- schema/model/main.tsp | 8 - .../.typra-generated/export-surfaces.json | 232 ----------- .../.typra-generated/hydration-seams.json | 64 --- .../tsp-output/.typra-generated/manifest.json | 40 -- .../tsp-output/.typra-generated/vectors.json | 57 --- .../docs/reference/LiveChatConformance.md | 40 -- 39 files changed, 699 insertions(+), 2346 deletions(-) delete mode 100644 runtime/csharp/Prompty.Core/Model/LiveChatConformance.cs delete mode 100644 runtime/go/prompty/model/live_chat_conformance.go delete mode 100644 runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LiveChatConformance.java delete mode 100644 runtime/python/prompty/prompty/model/_LiveChatConformance.py delete mode 100644 runtime/python/prompty/tests/model/_drill_replay.py delete mode 100644 runtime/python/prompty/tests/model/cassettes/openai_chat_drill.json delete mode 100644 runtime/python/prompty/tests/model/record_cassette.py delete mode 100644 runtime/rust/prompty/src/model/live_chat_conformance.rs delete mode 100644 runtime/swift/prompty-model/Sources/PromptyModel/live_chat_conformance.swift delete mode 100644 runtime/typescript/packages/core/src/model/live-chat-conformance.ts delete mode 100644 runtime/typescript/packages/core/tests/model/cassettes/openai_chat_drill.json delete mode 100644 runtime/typescript/packages/core/tests/model/drill-replay.ts delete mode 100644 schema/model/conformance/seams/live-chat-conformance.tsp delete mode 100644 schema/model/conformance/vectors/drill.tsp delete mode 100644 web/src/content/docs/reference/LiveChatConformance.md diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs index 889793989..422839d89 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs @@ -74,10 +74,7 @@ public static partial class VectorAdapters ["WireConformance.toRequest"] = new(WireInvoke, AlignNormalize), }; - public static IReadOnlyDictionary Waivers() => new Dictionary - { - ["LiveChatConformance.complete"] = "drill: consumer-owned transport-seam probe (Python/TS only)", - }; + public static IReadOnlyDictionary Waivers() => new Dictionary(); public static JsonNode? Doubles() => null; diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs index 655221cdc..603b74058 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs @@ -570,62 +570,7 @@ public async Task Vector15DiscoveryConformanceMapModelOpenaiModelFinetuneNoOwner } [Fact] - public async Task Vector16LiveChatConformanceCompleteOpenaiChatDrill() - { - string vectorJson = """ -{ - "name": "openai_chat_drill", - "stage": "drill", - "provider": "openai", - "input": { - "provider": "openai", - "model": "gpt-4o-mini", - "apiKey": { - "$env": "OPENAI_API_KEY" - }, - "endpoint": { - "$env": "OPENAI_BASE_URL" - }, - "messages": [ - { - "role": "user", - "content": "Say hello in exactly one word." - } - ], - "options": { - "temperature": 0, - "maxOutputTokens": 16 - } - }, - "exchange": { - "transport": { - "baseUrl": { - "$env": "OPENAI_BASE_URL" - } - }, - "cassette": { - "$json": "cassettes/openai_chat_drill.json" - }, - "planes": [ - "transport", - "wire", - "semantic" - ] - }, - "expected": { - "role": "assistant", - "contentNonEmpty": true, - "finishReasonInEnum": true - }, - "operation": "complete" -} -"""; - var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); - await VectorRunner.RunVector("LiveChatConformance", "complete", vector, false, Seam()); - } - - [Fact] - public async Task Vector17LoadConformanceLoadBasicLoad() + public async Task Vector16LoadConformanceLoadBasicLoad() { string vectorJson = """ { @@ -696,7 +641,7 @@ public async Task Vector17LoadConformanceLoadBasicLoad() } [Fact] - public async Task Vector18LoadConformanceLoadConnectionTypesLoad() + public async Task Vector17LoadConformanceLoadConnectionTypesLoad() { string vectorJson = """ { @@ -732,7 +677,7 @@ public async Task Vector18LoadConformanceLoadConnectionTypesLoad() } [Fact] - public async Task Vector19LoadConformanceLoadEmbeddingLoad() + public async Task Vector18LoadConformanceLoadEmbeddingLoad() { string vectorJson = """ { @@ -775,7 +720,7 @@ public async Task Vector19LoadConformanceLoadEmbeddingLoad() } [Fact] - public async Task Vector20LoadConformanceLoadEmptyFrontmatterBodyOnly() + public async Task Vector19LoadConformanceLoadEmptyFrontmatterBodyOnly() { string vectorJson = """ { @@ -802,7 +747,7 @@ public async Task Vector20LoadConformanceLoadEmptyFrontmatterBodyOnly() } [Fact] - public async Task Vector21LoadConformanceLoadEnvDefault() + public async Task Vector20LoadConformanceLoadEnvDefault() { string vectorJson = """ { @@ -838,7 +783,7 @@ public async Task Vector21LoadConformanceLoadEnvDefault() } [Fact] - public async Task Vector22LoadConformanceLoadEnvMissingError() + public async Task Vector21LoadConformanceLoadEnvMissingError() { string vectorJson = """ { @@ -869,7 +814,7 @@ public async Task Vector22LoadConformanceLoadEnvMissingError() } [Fact] - public async Task Vector23LoadConformanceLoadEnvResolution() + public async Task Vector22LoadConformanceLoadEnvResolution() { string vectorJson = """ { @@ -907,7 +852,7 @@ public async Task Vector23LoadConformanceLoadEnvResolution() } [Fact] - public async Task Vector24LoadConformanceLoadFileReferenceInTreeAllowed() + public async Task Vector23LoadConformanceLoadFileReferenceInTreeAllowed() { string vectorJson = """ { @@ -947,7 +892,7 @@ public async Task Vector24LoadConformanceLoadFileReferenceInTreeAllowed() } [Fact] - public async Task Vector25LoadConformanceLoadFileReferenceTraversalRejected() + public async Task Vector24LoadConformanceLoadFileReferenceTraversalRejected() { string vectorJson = """ { @@ -982,7 +927,7 @@ public async Task Vector25LoadConformanceLoadFileReferenceTraversalRejected() } [Fact] - public async Task Vector26LoadConformanceLoadFileResolution() + public async Task Vector25LoadConformanceLoadFileResolution() { string vectorJson = """ { @@ -1024,7 +969,7 @@ public async Task Vector26LoadConformanceLoadFileResolution() } [Fact] - public async Task Vector27LoadConformanceLoadImageApitypeLoad() + public async Task Vector26LoadConformanceLoadImageApitypeLoad() { string vectorJson = """ { @@ -1070,7 +1015,7 @@ public async Task Vector27LoadConformanceLoadImageApitypeLoad() } [Fact] - public async Task Vector28LoadConformanceLoadInputScalarShorthand() + public async Task Vector27LoadConformanceLoadInputScalarShorthand() { string vectorJson = """ { @@ -1117,7 +1062,7 @@ public async Task Vector28LoadConformanceLoadInputScalarShorthand() } [Fact] - public async Task Vector29LoadConformanceLoadInputValidationDefaultFill() + public async Task Vector28LoadConformanceLoadInputValidationDefaultFill() { string vectorJson = """ { @@ -1151,7 +1096,7 @@ public async Task Vector29LoadConformanceLoadInputValidationDefaultFill() } [Fact] - public async Task Vector30LoadConformanceLoadInputValidationExampleNotUsed() + public async Task Vector29LoadConformanceLoadInputValidationExampleNotUsed() { string vectorJson = """ { @@ -1184,7 +1129,7 @@ public async Task Vector30LoadConformanceLoadInputValidationExampleNotUsed() } [Fact] - public async Task Vector31LoadConformanceLoadInputValidationOptionalOmit() + public async Task Vector30LoadConformanceLoadInputValidationOptionalOmit() { string vectorJson = """ { @@ -1216,7 +1161,7 @@ public async Task Vector31LoadConformanceLoadInputValidationOptionalOmit() } [Fact] - public async Task Vector32LoadConformanceLoadInputValidationRequired() + public async Task Vector31LoadConformanceLoadInputValidationRequired() { string vectorJson = """ { @@ -1249,7 +1194,7 @@ public async Task Vector32LoadConformanceLoadInputValidationRequired() } [Fact] - public async Task Vector33LoadConformanceLoadInstructionsFromBody() + public async Task Vector32LoadConformanceLoadInstructionsFromBody() { string vectorJson = """ { @@ -1274,7 +1219,7 @@ public async Task Vector33LoadConformanceLoadInstructionsFromBody() } [Fact] - public async Task Vector34LoadConformanceLoadInvalidFrontmatterError() + public async Task Vector33LoadConformanceLoadInvalidFrontmatterError() { string vectorJson = """ { @@ -1295,7 +1240,7 @@ public async Task Vector34LoadConformanceLoadInvalidFrontmatterError() } [Fact] - public async Task Vector35LoadConformanceLoadKindAlwaysPrompt() + public async Task Vector34LoadConformanceLoadKindAlwaysPrompt() { string vectorJson = """ { @@ -1316,7 +1261,7 @@ public async Task Vector35LoadConformanceLoadKindAlwaysPrompt() } [Fact] - public async Task Vector36LoadConformanceLoadMinimalLoad() + public async Task Vector35LoadConformanceLoadMinimalLoad() { string vectorJson = """ { @@ -1345,7 +1290,7 @@ public async Task Vector36LoadConformanceLoadMinimalLoad() } [Fact] - public async Task Vector37LoadConformanceLoadMissingFileError() + public async Task Vector36LoadConformanceLoadMissingFileError() { string vectorJson = """ { @@ -1366,7 +1311,7 @@ public async Task Vector37LoadConformanceLoadMissingFileError() } [Fact] - public async Task Vector38LoadConformanceLoadModelShorthand() + public async Task Vector37LoadConformanceLoadModelShorthand() { string vectorJson = """ { @@ -1394,7 +1339,7 @@ public async Task Vector38LoadConformanceLoadModelShorthand() } [Fact] - public async Task Vector39LoadConformanceLoadStructuredOutputsLoad() + public async Task Vector38LoadConformanceLoadStructuredOutputsLoad() { string vectorJson = """ { @@ -1462,7 +1407,7 @@ public async Task Vector39LoadConformanceLoadStructuredOutputsLoad() } [Fact] - public async Task Vector40LoadConformanceLoadTemplateStringInvalid() + public async Task Vector39LoadConformanceLoadTemplateStringInvalid() { string vectorJson = """ { @@ -1489,7 +1434,7 @@ public async Task Vector40LoadConformanceLoadTemplateStringInvalid() } [Fact] - public async Task Vector41LoadConformanceLoadToolsCustomLoad() + public async Task Vector40LoadConformanceLoadToolsCustomLoad() { string vectorJson = """ { @@ -1528,7 +1473,7 @@ public async Task Vector41LoadConformanceLoadToolsCustomLoad() } [Fact] - public async Task Vector42LoadConformanceLoadToolsFunctionLoad() + public async Task Vector41LoadConformanceLoadToolsFunctionLoad() { string vectorJson = """ { @@ -1596,7 +1541,7 @@ public async Task Vector42LoadConformanceLoadToolsFunctionLoad() } [Fact] - public async Task Vector43LoadConformanceLoadToolsMcpLoad() + public async Task Vector42LoadConformanceLoadToolsMcpLoad() { string vectorJson = """ { @@ -1637,7 +1582,7 @@ public async Task Vector43LoadConformanceLoadToolsMcpLoad() } [Fact] - public async Task Vector44LoadConformanceLoadToolsOpenapiLoad() + public async Task Vector43LoadConformanceLoadToolsOpenapiLoad() { string vectorJson = """ { @@ -1678,7 +1623,7 @@ public async Task Vector44LoadConformanceLoadToolsOpenapiLoad() } [Fact] - public async Task Vector45TurnConformanceReplayMaxIterations() + public async Task Vector44TurnConformanceReplayMaxIterations() { string vectorJson = """ { @@ -1715,7 +1660,7 @@ public async Task Vector45TurnConformanceReplayMaxIterations() } [Fact] - public async Task Vector46TurnConformanceReplayNoTool() + public async Task Vector45TurnConformanceReplayNoTool() { string vectorJson = """ { @@ -1748,7 +1693,7 @@ public async Task Vector46TurnConformanceReplayNoTool() } [Fact] - public async Task Vector47TurnConformanceReplayPermissionDenied() + public async Task Vector46TurnConformanceReplayPermissionDenied() { string vectorJson = """ { @@ -1784,7 +1729,7 @@ public async Task Vector47TurnConformanceReplayPermissionDenied() } [Fact] - public async Task Vector48TurnConformanceReplayToolFailure() + public async Task Vector47TurnConformanceReplayToolFailure() { string vectorJson = """ { @@ -1822,7 +1767,7 @@ public async Task Vector48TurnConformanceReplayToolFailure() } [Fact] - public async Task Vector49TurnConformanceReplayToolSuccess() + public async Task Vector48TurnConformanceReplayToolSuccess() { string vectorJson = """ { @@ -1860,7 +1805,7 @@ public async Task Vector49TurnConformanceReplayToolSuccess() } [Fact] - public async Task Vector50TurnConformanceRunAssistantToolCallsMetadata() + public async Task Vector49TurnConformanceRunAssistantToolCallsMetadata() { string vectorJson = """ { @@ -2014,7 +1959,7 @@ public async Task Vector50TurnConformanceRunAssistantToolCallsMetadata() } [Fact] - public async Task Vector51TurnConformanceRunAsyncToolFunction() + public async Task Vector50TurnConformanceRunAsyncToolFunction() { string vectorJson = """ { @@ -2126,7 +2071,7 @@ public async Task Vector51TurnConformanceRunAsyncToolFunction() } [Fact] - public async Task Vector52TurnConformanceRunBindingsInjected() + public async Task Vector51TurnConformanceRunBindingsInjected() { string vectorJson = """ { @@ -2258,7 +2203,7 @@ public async Task Vector52TurnConformanceRunBindingsInjected() } [Fact] - public async Task Vector53TurnConformanceRunCancellationBeforeLlm() + public async Task Vector52TurnConformanceRunCancellationBeforeLlm() { string vectorJson = """ { @@ -2321,7 +2266,7 @@ public async Task Vector53TurnConformanceRunCancellationBeforeLlm() } [Fact] - public async Task Vector54TurnConformanceRunCancellationBetweenIterations() + public async Task Vector53TurnConformanceRunCancellationBetweenIterations() { string vectorJson = """ { @@ -2453,7 +2398,7 @@ public async Task Vector54TurnConformanceRunCancellationBetweenIterations() } [Fact] - public async Task Vector55TurnConformanceRunCancellationBetweenTools() + public async Task Vector54TurnConformanceRunCancellationBetweenTools() { string vectorJson = """ { @@ -2594,7 +2539,7 @@ public async Task Vector55TurnConformanceRunCancellationBetweenTools() } [Fact] - public async Task Vector56TurnConformanceRunContextNoTrimWhenFits() + public async Task Vector55TurnConformanceRunContextNoTrimWhenFits() { string vectorJson = """ { @@ -2667,7 +2612,7 @@ public async Task Vector56TurnConformanceRunContextNoTrimWhenFits() } [Fact] - public async Task Vector57TurnConformanceRunContextPreservesSystemMessages() + public async Task Vector56TurnConformanceRunContextPreservesSystemMessages() { string vectorJson = """ { @@ -2776,7 +2721,7 @@ public async Task Vector57TurnConformanceRunContextPreservesSystemMessages() } [Fact] - public async Task Vector58TurnConformanceRunContextTrimBasic() + public async Task Vector57TurnConformanceRunContextTrimBasic() { string vectorJson = """ { @@ -2936,7 +2881,7 @@ public async Task Vector58TurnConformanceRunContextTrimBasic() } [Fact] - public async Task Vector59TurnConformanceRunEmptyToolResult() + public async Task Vector58TurnConformanceRunEmptyToolResult() { string vectorJson = """ { @@ -3053,7 +2998,7 @@ public async Task Vector59TurnConformanceRunEmptyToolResult() } [Fact] - public async Task Vector60TurnConformanceRunEventsBasicToolLoop() + public async Task Vector59TurnConformanceRunEventsBasicToolLoop() { string vectorJson = """ { @@ -3201,7 +3146,7 @@ public async Task Vector60TurnConformanceRunEventsBasicToolLoop() } [Fact] - public async Task Vector61TurnConformanceRunEventsErrorLogged() + public async Task Vector60TurnConformanceRunEventsErrorLogged() { string vectorJson = """ { @@ -3329,7 +3274,7 @@ public async Task Vector61TurnConformanceRunEventsErrorLogged() } [Fact] - public async Task Vector62TurnConformanceRunEventsNoTools() + public async Task Vector61TurnConformanceRunEventsNoTools() { string vectorJson = """ { @@ -3409,7 +3354,7 @@ public async Task Vector62TurnConformanceRunEventsNoTools() } [Fact] - public async Task Vector63TurnConformanceRunGuardrailAllPass() + public async Task Vector62TurnConformanceRunGuardrailAllPass() { string vectorJson = """ { @@ -3534,7 +3479,7 @@ public async Task Vector63TurnConformanceRunGuardrailAllPass() } [Fact] - public async Task Vector64TurnConformanceRunGuardrailInputDeny() + public async Task Vector63TurnConformanceRunGuardrailInputDeny() { string vectorJson = """ { @@ -3592,7 +3537,7 @@ public async Task Vector64TurnConformanceRunGuardrailInputDeny() } [Fact] - public async Task Vector65TurnConformanceRunGuardrailOutputDeny() + public async Task Vector64TurnConformanceRunGuardrailOutputDeny() { string vectorJson = """ { @@ -3672,7 +3617,7 @@ public async Task Vector65TurnConformanceRunGuardrailOutputDeny() } [Fact] - public async Task Vector66TurnConformanceRunGuardrailToolDeny() + public async Task Vector65TurnConformanceRunGuardrailToolDeny() { string vectorJson = """ { @@ -3828,7 +3773,7 @@ public async Task Vector66TurnConformanceRunGuardrailToolDeny() } [Fact] - public async Task Vector67TurnConformanceRunMaxIterationsExceeded() + public async Task Vector66TurnConformanceRunMaxIterationsExceeded() { string vectorJson = """ { @@ -4340,7 +4285,7 @@ public async Task Vector67TurnConformanceRunMaxIterationsExceeded() } [Fact] - public async Task Vector68TurnConformanceRunMultiTurnToolCalls() + public async Task Vector67TurnConformanceRunMultiTurnToolCalls() { string vectorJson = """ { @@ -4519,7 +4464,7 @@ public async Task Vector68TurnConformanceRunMultiTurnToolCalls() } [Fact] - public async Task Vector69TurnConformanceRunMultipleToolCallsSingleTurn() + public async Task Vector68TurnConformanceRunMultipleToolCallsSingleTurn() { string vectorJson = """ { @@ -4702,7 +4647,7 @@ public async Task Vector69TurnConformanceRunMultipleToolCallsSingleTurn() } [Fact] - public async Task Vector70TurnConformanceRunNoToolCalls() + public async Task Vector69TurnConformanceRunNoToolCalls() { string vectorJson = """ { @@ -4772,7 +4717,7 @@ public async Task Vector70TurnConformanceRunNoToolCalls() } [Fact] - public async Task Vector71TurnConformanceRunParallelToolsBasic() + public async Task Vector70TurnConformanceRunParallelToolsBasic() { string vectorJson = """ { @@ -4948,7 +4893,7 @@ public async Task Vector71TurnConformanceRunParallelToolsBasic() } [Fact] - public async Task Vector72TurnConformanceRunParallelToolsWithGuardrailDeny() + public async Task Vector71TurnConformanceRunParallelToolsWithGuardrailDeny() { string vectorJson = """ { @@ -5142,7 +5087,7 @@ public async Task Vector72TurnConformanceRunParallelToolsWithGuardrailDeny() } [Fact] - public async Task Vector73TurnConformanceRunSingleToolCall() + public async Task Vector72TurnConformanceRunSingleToolCall() { string vectorJson = """ { @@ -5291,7 +5236,7 @@ public async Task Vector73TurnConformanceRunSingleToolCall() } [Fact] - public async Task Vector74TurnConformanceRunSteeringInjectMessage() + public async Task Vector73TurnConformanceRunSteeringInjectMessage() { string vectorJson = """ { @@ -5501,7 +5446,7 @@ public async Task Vector74TurnConformanceRunSteeringInjectMessage() } [Fact] - public async Task Vector75TurnConformanceRunSteeringMultipleMessages() + public async Task Vector74TurnConformanceRunSteeringMultipleMessages() { string vectorJson = """ { @@ -5673,7 +5618,7 @@ public async Task Vector75TurnConformanceRunSteeringMultipleMessages() } [Fact] - public async Task Vector76TurnConformanceRunToolNotRegisteredError() + public async Task Vector75TurnConformanceRunToolNotRegisteredError() { string vectorJson = """ { @@ -5759,7 +5704,7 @@ public async Task Vector76TurnConformanceRunToolNotRegisteredError() } [Fact] - public async Task Vector77TurnConformanceRunToolResultMessageFormat() + public async Task Vector76TurnConformanceRunToolResultMessageFormat() { string vectorJson = """ { @@ -5882,7 +5827,7 @@ public async Task Vector77TurnConformanceRunToolResultMessageFormat() } [Fact] - public async Task Vector78TurnConformanceRunTurnCancelBeforeContext() + public async Task Vector77TurnConformanceRunTurnCancelBeforeContext() { string vectorJson = """ { @@ -5916,7 +5861,7 @@ public async Task Vector78TurnConformanceRunTurnCancelBeforeContext() } [Fact] - public async Task Vector79TurnConformanceRunTurnDelegatedProviderState() + public async Task Vector78TurnConformanceRunTurnDelegatedProviderState() { string vectorJson = """ { @@ -5985,7 +5930,7 @@ public async Task Vector79TurnConformanceRunTurnDelegatedProviderState() } [Fact] - public async Task Vector80TurnConformanceRunTurnFinalOutput() + public async Task Vector79TurnConformanceRunTurnFinalOutput() { string vectorJson = """ { @@ -6032,7 +5977,7 @@ public async Task Vector80TurnConformanceRunTurnFinalOutput() } [Fact] - public async Task Vector81TurnConformanceRunTurnOrderedToolRound() + public async Task Vector80TurnConformanceRunTurnOrderedToolRound() { string vectorJson = """ { @@ -6125,7 +6070,7 @@ public async Task Vector81TurnConformanceRunTurnOrderedToolRound() } [Fact] - public async Task Vector82TurnConformanceRunTurnPermissionDenialIsModelVisible() + public async Task Vector81TurnConformanceRunTurnPermissionDenialIsModelVisible() { string vectorJson = """ { @@ -6178,7 +6123,7 @@ public async Task Vector82TurnConformanceRunTurnPermissionDenialIsModelVisible() } [Fact] - public async Task Vector83WireConformanceToRequestAnthropicImageFormat() + public async Task Vector82WireConformanceToRequestAnthropicImageFormat() { string vectorJson = """ { @@ -6250,7 +6195,7 @@ public async Task Vector83WireConformanceToRequestAnthropicImageFormat() } [Fact] - public async Task Vector84WireConformanceToRequestAnthropicMaxTokensRequired() + public async Task Vector83WireConformanceToRequestAnthropicMaxTokensRequired() { string vectorJson = """ { @@ -6307,7 +6252,7 @@ public async Task Vector84WireConformanceToRequestAnthropicMaxTokensRequired() } [Fact] - public async Task Vector85WireConformanceToRequestAnthropicOptions() + public async Task Vector84WireConformanceToRequestAnthropicOptions() { string vectorJson = """ { @@ -6378,7 +6323,7 @@ public async Task Vector85WireConformanceToRequestAnthropicOptions() } [Fact] - public async Task Vector86WireConformanceToRequestAnthropicSystemSeparate() + public async Task Vector85WireConformanceToRequestAnthropicSystemSeparate() { string vectorJson = """ { @@ -6447,7 +6392,7 @@ public async Task Vector86WireConformanceToRequestAnthropicSystemSeparate() } [Fact] - public async Task Vector87WireConformanceToRequestAnthropicToolWire() + public async Task Vector86WireConformanceToRequestAnthropicToolWire() { string vectorJson = """ { @@ -6534,7 +6479,7 @@ public async Task Vector87WireConformanceToRequestAnthropicToolWire() } [Fact] - public async Task Vector88WireConformanceToRequestChatAudioMp3() + public async Task Vector87WireConformanceToRequestChatAudioMp3() { string vectorJson = """ { @@ -6594,7 +6539,7 @@ public async Task Vector88WireConformanceToRequestChatAudioMp3() } [Fact] - public async Task Vector89WireConformanceToRequestChatAudioPart() + public async Task Vector88WireConformanceToRequestChatAudioPart() { string vectorJson = """ { @@ -6654,7 +6599,7 @@ public async Task Vector89WireConformanceToRequestChatAudioPart() } [Fact] - public async Task Vector90WireConformanceToRequestChatImageBase64() + public async Task Vector89WireConformanceToRequestChatImageBase64() { string vectorJson = """ { @@ -6720,7 +6665,7 @@ public async Task Vector90WireConformanceToRequestChatImageBase64() } [Fact] - public async Task Vector91WireConformanceToRequestChatImagePart() + public async Task Vector90WireConformanceToRequestChatImagePart() { string vectorJson = """ { @@ -6786,7 +6731,7 @@ public async Task Vector91WireConformanceToRequestChatImagePart() } [Fact] - public async Task Vector92WireConformanceToRequestChatMultipartContent() + public async Task Vector91WireConformanceToRequestChatMultipartContent() { string vectorJson = """ { @@ -6852,7 +6797,7 @@ public async Task Vector92WireConformanceToRequestChatMultipartContent() } [Fact] - public async Task Vector93WireConformanceToRequestChatSimple() + public async Task Vector92WireConformanceToRequestChatSimple() { string vectorJson = """ { @@ -6916,7 +6861,7 @@ public async Task Vector93WireConformanceToRequestChatSimple() } [Fact] - public async Task Vector94WireConformanceToRequestChatSingleTextOptimized() + public async Task Vector93WireConformanceToRequestChatSingleTextOptimized() { string vectorJson = """ { @@ -6967,7 +6912,7 @@ public async Task Vector94WireConformanceToRequestChatSingleTextOptimized() } [Fact] - public async Task Vector95WireConformanceToRequestChatWithOptions() + public async Task Vector94WireConformanceToRequestChatWithOptions() { string vectorJson = """ { @@ -7037,7 +6982,7 @@ public async Task Vector95WireConformanceToRequestChatWithOptions() } [Fact] - public async Task Vector96WireConformanceToRequestEmbeddingWire() + public async Task Vector95WireConformanceToRequestEmbeddingWire() { string vectorJson = """ { @@ -7083,7 +7028,7 @@ public async Task Vector96WireConformanceToRequestEmbeddingWire() } [Fact] - public async Task Vector97WireConformanceToRequestImageWire() + public async Task Vector96WireConformanceToRequestImageWire() { string vectorJson = """ { @@ -7129,7 +7074,7 @@ public async Task Vector97WireConformanceToRequestImageWire() } [Fact] - public async Task Vector98WireConformanceToRequestKindToJsonTypeMapping() + public async Task Vector97WireConformanceToRequestKindToJsonTypeMapping() { string vectorJson = """ { @@ -7250,7 +7195,7 @@ public async Task Vector98WireConformanceToRequestKindToJsonTypeMapping() } [Fact] - public async Task Vector99WireConformanceToRequestOptionsAdditionalProperties() + public async Task Vector98WireConformanceToRequestOptionsAdditionalProperties() { string vectorJson = """ { @@ -7306,7 +7251,7 @@ public async Task Vector99WireConformanceToRequestOptionsAdditionalProperties() } [Fact] - public async Task Vector100WireConformanceToRequestOptionsMaxCompletionTokens() + public async Task Vector99WireConformanceToRequestOptionsMaxCompletionTokens() { string vectorJson = """ { @@ -7360,7 +7305,7 @@ public async Task Vector100WireConformanceToRequestOptionsMaxCompletionTokens() } [Fact] - public async Task Vector101WireConformanceToRequestOptionsStopSequences() + public async Task Vector100WireConformanceToRequestOptionsStopSequences() { string vectorJson = """ { @@ -7420,7 +7365,7 @@ public async Task Vector101WireConformanceToRequestOptionsStopSequences() } [Fact] - public async Task Vector102WireConformanceToRequestResponsesSimple() + public async Task Vector101WireConformanceToRequestResponsesSimple() { string vectorJson = """ { @@ -7481,7 +7426,7 @@ public async Task Vector102WireConformanceToRequestResponsesSimple() } [Fact] - public async Task Vector103WireConformanceToRequestResponsesStructuredOutput() + public async Task Vector102WireConformanceToRequestResponsesStructuredOutput() { string vectorJson = """ { @@ -7570,7 +7515,7 @@ public async Task Vector103WireConformanceToRequestResponsesStructuredOutput() } [Fact] - public async Task Vector104WireConformanceToRequestResponsesWithTools() + public async Task Vector103WireConformanceToRequestResponsesWithTools() { string vectorJson = """ { @@ -7652,7 +7597,7 @@ public async Task Vector104WireConformanceToRequestResponsesWithTools() } [Fact] - public async Task Vector105WireConformanceToRequestStructuredOutput() + public async Task Vector104WireConformanceToRequestStructuredOutput() { string vectorJson = """ { @@ -7737,7 +7682,7 @@ public async Task Vector105WireConformanceToRequestStructuredOutput() } [Fact] - public async Task Vector106WireConformanceToRequestStructuredOutputNestedOptional() + public async Task Vector105WireConformanceToRequestStructuredOutputNestedOptional() { string vectorJson = """ { @@ -7841,7 +7786,7 @@ public async Task Vector106WireConformanceToRequestStructuredOutputNestedOptiona } [Fact] - public async Task Vector107WireConformanceToRequestToolsBindingsStripped() + public async Task Vector106WireConformanceToRequestToolsBindingsStripped() { string vectorJson = """ { @@ -7935,7 +7880,7 @@ public async Task Vector107WireConformanceToRequestToolsBindingsStripped() } [Fact] - public async Task Vector108WireConformanceToRequestToolsFunctionWire() + public async Task Vector107WireConformanceToRequestToolsFunctionWire() { string vectorJson = """ { @@ -8019,7 +7964,7 @@ public async Task Vector108WireConformanceToRequestToolsFunctionWire() } [Fact] - public async Task Vector109WireConformanceToRequestToolsNullWhenEmpty() + public async Task Vector108WireConformanceToRequestToolsNullWhenEmpty() { string vectorJson = """ { @@ -8070,7 +8015,7 @@ public async Task Vector109WireConformanceToRequestToolsNullWhenEmpty() } [Fact] - public async Task Vector110WireConformanceToRequestToolsStrictMode() + public async Task Vector109WireConformanceToRequestToolsStrictMode() { string vectorJson = """ { diff --git a/runtime/csharp/Prompty.Core/Model/LiveChatConformance.cs b/runtime/csharp/Prompty.Core/Model/LiveChatConformance.cs deleted file mode 100644 index a6f614380..000000000 --- a/runtime/csharp/Prompty.Core/Model/LiveChatConformance.cs +++ /dev/null @@ -1,31 +0,0 @@ -// -// Copyright (c) Microsoft. All rights reserved. -#nullable enable - -using System.Threading; -using System.Threading.Tasks; - -#pragma warning disable IDE0130 -namespace Prompty.Core; -#pragma warning restore IDE0130 - - /// - /// Conformance-only seam hosting the "service drill" vectors: a conformance - /// - /// vector that crosses the REAL Executor->Processor transport seam instead of - /// - /// stopping at the SDK edge. A drill runs against a live provider when - /// - /// credentials are present and REPLAYS a recorded cassette when they are not; - /// - /// the live<->replay parity is the gate. - /// - /// Like every other conformance seam this interface exists purely to carry - /// - public interface ILiveChatConformance - { - /// - /// Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes - /// - Task CompleteAsync(object input); - } diff --git a/runtime/go/prompty/model/live_chat_conformance.go b/runtime/go/prompty/model/live_chat_conformance.go deleted file mode 100644 index b545fe189..000000000 --- a/runtime/go/prompty/model/live_chat_conformance.go +++ /dev/null @@ -1,17 +0,0 @@ -// -// Code generated by Typra emitter; DO NOT EDIT. - -package prompty - -// LiveChatConformance represents Conformance-only seam hosting the "service drill" vectors: a conformance -// vector that crosses the REAL Executor->Processor transport seam instead of -// stopping at the SDK edge. A drill runs against a live provider when -// credentials are present and REPLAYS a recorded cassette when they are not; -// the live<->replay parity is the gate. -// -// Like every other conformance seam this interface exists purely to carry - -type LiveChatConformance interface { - // Complete — Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes - Complete(input interface{}) (interface{}, error) -} diff --git a/runtime/go/prompty/model/vector_conformance_test.go b/runtime/go/prompty/model/vector_conformance_test.go index 09d5a5b55..0c8687e17 100644 --- a/runtime/go/prompty/model/vector_conformance_test.go +++ b/runtime/go/prompty/model/vector_conformance_test.go @@ -560,61 +560,7 @@ func TestVector15DiscoveryConformanceMapModelOpenaiModelFinetuneNoOwner(t *testi vectorrunner.RunVector(t, "DiscoveryConformance", "mapModel", vector, vcSeam()) } -func TestVector16LiveChatConformanceCompleteOpenaiChatDrill(t *testing.T) { - vectorJSON := `{ - "name": "openai_chat_drill", - "stage": "drill", - "provider": "openai", - "input": { - "provider": "openai", - "model": "gpt-4o-mini", - "apiKey": { - "$env": "OPENAI_API_KEY" - }, - "endpoint": { - "$env": "OPENAI_BASE_URL" - }, - "messages": [ - { - "role": "user", - "content": "Say hello in exactly one word." - } - ], - "options": { - "temperature": 0, - "maxOutputTokens": 16 - } - }, - "exchange": { - "transport": { - "baseUrl": { - "$env": "OPENAI_BASE_URL" - } - }, - "cassette": { - "$json": "cassettes/openai_chat_drill.json" - }, - "planes": [ - "transport", - "wire", - "semantic" - ] - }, - "expected": { - "role": "assistant", - "contentNonEmpty": true, - "finishReasonInEnum": true - }, - "operation": "complete" -}` - var vector map[string]any - if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { - t.Fatalf("failed to decode vector: %v", err) - } - vectorrunner.RunVector(t, "LiveChatConformance", "complete", vector, vcSeam()) -} - -func TestVector17LoadConformanceLoadBasicLoad(t *testing.T) { +func TestVector16LoadConformanceLoadBasicLoad(t *testing.T) { vectorJSON := `{ "name": "basic_load", "description": "Load basic.prompty with env var resolution and verify all PromptAgent fields", @@ -684,7 +630,7 @@ func TestVector17LoadConformanceLoadBasicLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector18LoadConformanceLoadConnectionTypesLoad(t *testing.T) { +func TestVector17LoadConformanceLoadConnectionTypesLoad(t *testing.T) { vectorJSON := `{ "name": "connection_types_load", "description": "All connection kinds (key, reference, remote, anonymous) load correctly", @@ -719,7 +665,7 @@ func TestVector18LoadConformanceLoadConnectionTypesLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector19LoadConformanceLoadEmbeddingLoad(t *testing.T) { +func TestVector18LoadConformanceLoadEmbeddingLoad(t *testing.T) { vectorJSON := `{ "name": "embedding_load", "description": "Load embedding.prompty and verify apiType=embedding with correct model id", @@ -761,7 +707,7 @@ func TestVector19LoadConformanceLoadEmbeddingLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector20LoadConformanceLoadEmptyFrontmatterBodyOnly(t *testing.T) { +func TestVector19LoadConformanceLoadEmptyFrontmatterBodyOnly(t *testing.T) { vectorJSON := `{ "name": "empty_frontmatter_body_only", "description": "Empty frontmatter with body — instructions extracted, defaults applied", @@ -787,7 +733,7 @@ func TestVector20LoadConformanceLoadEmptyFrontmatterBodyOnly(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector21LoadConformanceLoadEnvDefault(t *testing.T) { +func TestVector20LoadConformanceLoadEnvDefault(t *testing.T) { vectorJSON := `{ "name": "env_default", "description": "${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set", @@ -822,7 +768,7 @@ func TestVector21LoadConformanceLoadEnvDefault(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector22LoadConformanceLoadEnvMissingError(t *testing.T) { +func TestVector21LoadConformanceLoadEnvMissingError(t *testing.T) { vectorJSON := `{ "name": "env_missing_error", "description": "${env:NONEXISTENT} with no default and var not set raises an error", @@ -852,7 +798,7 @@ func TestVector22LoadConformanceLoadEnvMissingError(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector23LoadConformanceLoadEnvResolution(t *testing.T) { +func TestVector22LoadConformanceLoadEnvResolution(t *testing.T) { vectorJSON := `{ "name": "env_resolution", "description": "${env:MY_VAR} resolves to the environment variable value when set", @@ -889,7 +835,7 @@ func TestVector23LoadConformanceLoadEnvResolution(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector24LoadConformanceLoadFileReferenceInTreeAllowed(t *testing.T) { +func TestVector23LoadConformanceLoadFileReferenceInTreeAllowed(t *testing.T) { vectorJSON := `{ "name": "file_reference_in_tree_allowed", "description": "A ${file:sub/data.json} reference that stays within the prompt file's directory tree resolves to the parsed content (containment control alongside the traversal-reject vector)", @@ -928,7 +874,7 @@ func TestVector24LoadConformanceLoadFileReferenceInTreeAllowed(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector25LoadConformanceLoadFileReferenceTraversalRejected(t *testing.T) { +func TestVector24LoadConformanceLoadFileReferenceTraversalRejected(t *testing.T) { vectorJSON := "{\n \"name\": \"file_reference_traversal_rejected\",\n \"description\": \"A ${file:../secret.json} reference that escapes the prompt file's directory via `..` is rejected even though the target file exists, preventing arbitrary file disclosure (GHSA-7pfc-5v9r-j299). Uses forward-slash separators so the vector holds on every OS.\",\n \"stage\": \"load\",\n \"input\": {\n \"agent_subdir\": \"app\",\n \"frontmatter\": {\n \"name\": \"file-traversal-test\",\n \"model\": {\n \"id\": \"gpt-4\",\n \"connection\": \"${file:../secret.json}\"\n }\n },\n \"files\": {\n \"../secret.json\": {\n \"kind\": \"key\",\n \"endpoint\": \"https://attacker.example.com\",\n \"apiKey\": \"leaked\"\n }\n }\n },\n \"expectedError\": {\n \"kind\": \"file_reference\"\n },\n \"operation\": \"load\"\n}" var vector map[string]any if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { @@ -937,7 +883,7 @@ func TestVector25LoadConformanceLoadFileReferenceTraversalRejected(t *testing.T) vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector26LoadConformanceLoadFileResolution(t *testing.T) { +func TestVector25LoadConformanceLoadFileResolution(t *testing.T) { vectorJSON := `{ "name": "file_resolution", "description": "${file:shared_connection.json} resolves to the parsed JSON content of the referenced file", @@ -978,7 +924,7 @@ func TestVector26LoadConformanceLoadFileResolution(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector27LoadConformanceLoadImageApitypeLoad(t *testing.T) { +func TestVector26LoadConformanceLoadImageApitypeLoad(t *testing.T) { vectorJSON := `{ "name": "image_apitype_load", "description": "An image-generation prompt loads with apiType image and passthrough additionalProperties model options", @@ -1023,7 +969,7 @@ func TestVector27LoadConformanceLoadImageApitypeLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector28LoadConformanceLoadInputScalarShorthand(t *testing.T) { +func TestVector27LoadConformanceLoadInputScalarShorthand(t *testing.T) { vectorJSON := `{ "name": "input_scalar_shorthand", "description": "Scalar input values create typed Property with default set", @@ -1069,7 +1015,7 @@ func TestVector28LoadConformanceLoadInputScalarShorthand(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector29LoadConformanceLoadInputValidationDefaultFill(t *testing.T) { +func TestVector28LoadConformanceLoadInputValidationDefaultFill(t *testing.T) { vectorJSON := `{ "name": "input_validation_default_fill", "description": "An input with a default value is filled in when no value is provided", @@ -1102,7 +1048,7 @@ func TestVector29LoadConformanceLoadInputValidationDefaultFill(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector30LoadConformanceLoadInputValidationExampleNotUsed(t *testing.T) { +func TestVector29LoadConformanceLoadInputValidationExampleNotUsed(t *testing.T) { vectorJSON := `{ "name": "input_validation_example_not_used", "description": "An input with example='sample' but no default must NOT use the example at runtime", @@ -1134,7 +1080,7 @@ func TestVector30LoadConformanceLoadInputValidationExampleNotUsed(t *testing.T) vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector31LoadConformanceLoadInputValidationOptionalOmit(t *testing.T) { +func TestVector30LoadConformanceLoadInputValidationOptionalOmit(t *testing.T) { vectorJSON := `{ "name": "input_validation_optional_omit", "description": "A non-required input with no default and no value is omitted (not an error)", @@ -1165,7 +1111,7 @@ func TestVector31LoadConformanceLoadInputValidationOptionalOmit(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector32LoadConformanceLoadInputValidationRequired(t *testing.T) { +func TestVector31LoadConformanceLoadInputValidationRequired(t *testing.T) { vectorJSON := `{ "name": "input_validation_required", "description": "A required input with no default and no value provided raises ValueError", @@ -1197,7 +1143,7 @@ func TestVector32LoadConformanceLoadInputValidationRequired(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector33LoadConformanceLoadInstructionsFromBody(t *testing.T) { +func TestVector32LoadConformanceLoadInstructionsFromBody(t *testing.T) { vectorJSON := `{ "name": "instructions_from_body", "description": "The markdown body after the closing --- becomes the instructions field", @@ -1221,7 +1167,7 @@ func TestVector33LoadConformanceLoadInstructionsFromBody(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector34LoadConformanceLoadInvalidFrontmatterError(t *testing.T) { +func TestVector33LoadConformanceLoadInvalidFrontmatterError(t *testing.T) { vectorJSON := `{ "name": "invalid_frontmatter_error", "description": "Malformed YAML in frontmatter raises a parse error", @@ -1241,7 +1187,7 @@ func TestVector34LoadConformanceLoadInvalidFrontmatterError(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector35LoadConformanceLoadKindAlwaysPrompt(t *testing.T) { +func TestVector34LoadConformanceLoadKindAlwaysPrompt(t *testing.T) { vectorJSON := `{ "name": "kind_always_prompt", "description": "The loader always injects kind='prompt' — .prompty files never specify kind themselves", @@ -1261,7 +1207,7 @@ func TestVector35LoadConformanceLoadKindAlwaysPrompt(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector36LoadConformanceLoadMinimalLoad(t *testing.T) { +func TestVector35LoadConformanceLoadMinimalLoad(t *testing.T) { vectorJSON := `{ "name": "minimal_load", "description": "Load minimal.prompty with model shorthand and bare-minimum frontmatter", @@ -1289,7 +1235,7 @@ func TestVector36LoadConformanceLoadMinimalLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector37LoadConformanceLoadMissingFileError(t *testing.T) { +func TestVector36LoadConformanceLoadMissingFileError(t *testing.T) { vectorJSON := `{ "name": "missing_file_error", "description": "Loading a nonexistent .prompty file raises FileNotFoundError", @@ -1309,7 +1255,7 @@ func TestVector37LoadConformanceLoadMissingFileError(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector38LoadConformanceLoadModelShorthand(t *testing.T) { +func TestVector37LoadConformanceLoadModelShorthand(t *testing.T) { vectorJSON := `{ "name": "model_shorthand", "description": "String shorthand 'model: gpt-4o' expands to a Model object with id set", @@ -1336,7 +1282,7 @@ func TestVector38LoadConformanceLoadModelShorthand(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector39LoadConformanceLoadStructuredOutputsLoad(t *testing.T) { +func TestVector38LoadConformanceLoadStructuredOutputsLoad(t *testing.T) { vectorJSON := `{ "name": "structured_outputs_load", "description": "Outputs load as a list of properties preserving kind, description, and required flags", @@ -1403,7 +1349,7 @@ func TestVector39LoadConformanceLoadStructuredOutputsLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector40LoadConformanceLoadTemplateStringInvalid(t *testing.T) { +func TestVector39LoadConformanceLoadTemplateStringInvalid(t *testing.T) { vectorJSON := `{ "name": "template_string_invalid", "description": "Template as a bare string is not valid v2 — must be an object with format/parser", @@ -1429,7 +1375,7 @@ func TestVector40LoadConformanceLoadTemplateStringInvalid(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector41LoadConformanceLoadToolsCustomLoad(t *testing.T) { +func TestVector40LoadConformanceLoadToolsCustomLoad(t *testing.T) { vectorJSON := `{ "name": "tools_custom_load", "description": "Load a prompty with an unknown tool kind — falls through to CustomTool", @@ -1467,7 +1413,7 @@ func TestVector41LoadConformanceLoadToolsCustomLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector42LoadConformanceLoadToolsFunctionLoad(t *testing.T) { +func TestVector41LoadConformanceLoadToolsFunctionLoad(t *testing.T) { vectorJSON := `{ "name": "tools_function_load", "description": "Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings", @@ -1534,7 +1480,7 @@ func TestVector42LoadConformanceLoadToolsFunctionLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector43LoadConformanceLoadToolsMcpLoad(t *testing.T) { +func TestVector42LoadConformanceLoadToolsMcpLoad(t *testing.T) { vectorJSON := `{ "name": "tools_mcp_load", "description": "Load a prompty with an MCP tool — kind, serverName, connection preserved", @@ -1574,7 +1520,7 @@ func TestVector43LoadConformanceLoadToolsMcpLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector44LoadConformanceLoadToolsOpenapiLoad(t *testing.T) { +func TestVector43LoadConformanceLoadToolsOpenapiLoad(t *testing.T) { vectorJSON := `{ "name": "tools_openapi_load", "description": "Load a prompty with an OpenAPI tool — specification path preserved", @@ -1614,7 +1560,7 @@ func TestVector44LoadConformanceLoadToolsOpenapiLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector45TurnConformanceReplayMaxIterations(t *testing.T) { +func TestVector44TurnConformanceReplayMaxIterations(t *testing.T) { vectorJSON := `{ "name": "max_iterations", "stage": "replay", @@ -1650,7 +1596,7 @@ func TestVector45TurnConformanceReplayMaxIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector46TurnConformanceReplayNoTool(t *testing.T) { +func TestVector45TurnConformanceReplayNoTool(t *testing.T) { vectorJSON := `{ "name": "no_tool", "stage": "replay", @@ -1682,7 +1628,7 @@ func TestVector46TurnConformanceReplayNoTool(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector47TurnConformanceReplayPermissionDenied(t *testing.T) { +func TestVector46TurnConformanceReplayPermissionDenied(t *testing.T) { vectorJSON := `{ "name": "permission_denied", "stage": "replay", @@ -1717,7 +1663,7 @@ func TestVector47TurnConformanceReplayPermissionDenied(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector48TurnConformanceReplayToolFailure(t *testing.T) { +func TestVector47TurnConformanceReplayToolFailure(t *testing.T) { vectorJSON := `{ "name": "tool_failure", "stage": "replay", @@ -1754,7 +1700,7 @@ func TestVector48TurnConformanceReplayToolFailure(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector49TurnConformanceReplayToolSuccess(t *testing.T) { +func TestVector48TurnConformanceReplayToolSuccess(t *testing.T) { vectorJSON := `{ "name": "tool_success", "stage": "replay", @@ -1791,7 +1737,7 @@ func TestVector49TurnConformanceReplayToolSuccess(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector50TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { +func TestVector49TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorJSON := `{ "name": "assistant_tool_calls_metadata", "description": "When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.", @@ -1944,7 +1890,7 @@ func TestVector50TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector51TurnConformanceRunAsyncToolFunction(t *testing.T) { +func TestVector50TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorJSON := `{ "name": "async_tool_function", "description": "Agent loop correctly handles async tool functions", @@ -2055,7 +2001,7 @@ func TestVector51TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector52TurnConformanceRunBindingsInjected(t *testing.T) { +func TestVector51TurnConformanceRunBindingsInjected(t *testing.T) { vectorJSON := `{ "name": "bindings_injected", "description": "Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.", @@ -2186,7 +2132,7 @@ func TestVector52TurnConformanceRunBindingsInjected(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector53TurnConformanceRunCancellationBeforeLlm(t *testing.T) { +func TestVector52TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorJSON := `{ "name": "cancellation_before_llm", "description": "§13.2 Cancellation — Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.", @@ -2248,7 +2194,7 @@ func TestVector53TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector54TurnConformanceRunCancellationBetweenIterations(t *testing.T) { +func TestVector53TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_iterations", "description": "§13.2 Cancellation — Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.", @@ -2379,7 +2325,7 @@ func TestVector54TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector55TurnConformanceRunCancellationBetweenTools(t *testing.T) { +func TestVector54TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_tools", "description": "§13.2 Cancellation — LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.", @@ -2519,7 +2465,7 @@ func TestVector55TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector56TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { +func TestVector55TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorJSON := `{ "name": "context_no_trim_when_fits", "description": "§13.3 Context Window — Messages fit within the context budget. No trimming occurs and all messages are preserved.", @@ -2591,7 +2537,7 @@ func TestVector56TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector57TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { +func TestVector56TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { vectorJSON := `{ "name": "context_preserves_system_messages", "description": "§13.3 Context Window — Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.", @@ -2699,7 +2645,7 @@ func TestVector57TurnConformanceRunContextPreservesSystemMessages(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector58TurnConformanceRunContextTrimBasic(t *testing.T) { +func TestVector57TurnConformanceRunContextTrimBasic(t *testing.T) { vectorJSON := `{ "name": "context_trim_basic", "description": "§13.3 Context Window — Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.", @@ -2858,7 +2804,7 @@ func TestVector58TurnConformanceRunContextTrimBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector59TurnConformanceRunEmptyToolResult(t *testing.T) { +func TestVector58TurnConformanceRunEmptyToolResult(t *testing.T) { vectorJSON := `{ "name": "empty_tool_result", "description": "Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content — it must not be skipped or filtered out.", @@ -2974,7 +2920,7 @@ func TestVector59TurnConformanceRunEmptyToolResult(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector60TurnConformanceRunEventsBasicToolLoop(t *testing.T) { +func TestVector59TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorJSON := `{ "name": "events_basic_tool_loop", "description": "§13.1 Events — A 2-turn tool-call loop emits the full event lifecycle: status → tool_call_start → tool_result → messages_updated → done.", @@ -3121,7 +3067,7 @@ func TestVector60TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector61TurnConformanceRunEventsErrorLogged(t *testing.T) { +func TestVector60TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorJSON := `{ "name": "events_error_logged", "description": "§13.1 Events — A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.", @@ -3248,7 +3194,7 @@ func TestVector61TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector62TurnConformanceRunEventsNoTools(t *testing.T) { +func TestVector61TurnConformanceRunEventsNoTools(t *testing.T) { vectorJSON := `{ "name": "events_no_tools", "description": "§13.1 Events — Single-turn completion with no tool calls. Only the done event is emitted.", @@ -3327,7 +3273,7 @@ func TestVector62TurnConformanceRunEventsNoTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector63TurnConformanceRunGuardrailAllPass(t *testing.T) { +func TestVector62TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorJSON := `{ "name": "guardrail_all_pass", "description": "§13.4 Guardrails — All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.", @@ -3451,7 +3397,7 @@ func TestVector63TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector64TurnConformanceRunGuardrailInputDeny(t *testing.T) { +func TestVector63TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_input_deny", "description": "§13.4 Guardrails — Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.", @@ -3508,7 +3454,7 @@ func TestVector64TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector65TurnConformanceRunGuardrailOutputDeny(t *testing.T) { +func TestVector64TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_output_deny", "description": "§13.4 Guardrails — Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.", @@ -3587,7 +3533,7 @@ func TestVector65TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector66TurnConformanceRunGuardrailToolDeny(t *testing.T) { +func TestVector65TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_tool_deny", "description": "§13.4 Guardrails — LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.", @@ -3742,7 +3688,7 @@ func TestVector66TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector67TurnConformanceRunMaxIterationsExceeded(t *testing.T) { +func TestVector66TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorJSON := `{ "name": "max_iterations_exceeded", "description": "LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.", @@ -4253,7 +4199,7 @@ func TestVector67TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector68TurnConformanceRunMultiTurnToolCalls(t *testing.T) { +func TestVector67TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorJSON := `{ "name": "multi_turn_tool_calls", "description": "Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.", @@ -4431,7 +4377,7 @@ func TestVector68TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector69TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { +func TestVector68TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorJSON := `{ "name": "multiple_tool_calls_single_turn", "description": "Turn 1: LLM returns two parallel tool calls — get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.", @@ -4613,7 +4559,7 @@ func TestVector69TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector70TurnConformanceRunNoToolCalls(t *testing.T) { +func TestVector69TurnConformanceRunNoToolCalls(t *testing.T) { vectorJSON := `{ "name": "no_tool_calls", "description": "LLM returns content immediately with no tool calls — the agent loop completes in a single iteration.", @@ -4682,7 +4628,7 @@ func TestVector70TurnConformanceRunNoToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector71TurnConformanceRunParallelToolsBasic(t *testing.T) { +func TestVector70TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_basic", "description": "§13.6 Parallel Tools — LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.", @@ -4857,7 +4803,7 @@ func TestVector71TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector72TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { +func TestVector71TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_guardrail_deny", "description": "§13.6 Parallel Tools + §13.4 Guardrails — 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.", @@ -5050,7 +4996,7 @@ func TestVector72TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector73TurnConformanceRunSingleToolCall(t *testing.T) { +func TestVector72TurnConformanceRunSingleToolCall(t *testing.T) { vectorJSON := `{ "name": "single_tool_call", "description": "Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.", @@ -5198,7 +5144,7 @@ func TestVector73TurnConformanceRunSingleToolCall(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector74TurnConformanceRunSteeringInjectMessage(t *testing.T) { +func TestVector73TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorJSON := `{ "name": "steering_inject_message", "description": "§13.5 Steering — After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.", @@ -5407,7 +5353,7 @@ func TestVector74TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector75TurnConformanceRunSteeringMultipleMessages(t *testing.T) { +func TestVector74TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorJSON := `{ "name": "steering_multiple_messages", "description": "§13.5 Steering — Two steering messages are queued and both injected before iteration 2. Tests atomic drain — both appear in order.", @@ -5578,7 +5524,7 @@ func TestVector75TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector76TurnConformanceRunToolNotRegisteredError(t *testing.T) { +func TestVector75TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorJSON := `{ "name": "tool_not_registered_error", "description": "LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.", @@ -5663,7 +5609,7 @@ func TestVector76TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector77TurnConformanceRunToolResultMessageFormat(t *testing.T) { +func TestVector76TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorJSON := `{ "name": "tool_result_message_format", "description": "After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.", @@ -5785,7 +5731,7 @@ func TestVector77TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector78TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { +func TestVector77TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorJSON := `{ "name": "cancel_before_context", "stage": "turn", @@ -5818,7 +5764,7 @@ func TestVector78TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector79TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { +func TestVector78TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorJSON := `{ "name": "delegated_provider_state", "stage": "turn", @@ -5886,7 +5832,7 @@ func TestVector79TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector80TurnConformanceRunTurnFinalOutput(t *testing.T) { +func TestVector79TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorJSON := `{ "name": "final_output", "stage": "turn", @@ -5932,7 +5878,7 @@ func TestVector80TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector81TurnConformanceRunTurnOrderedToolRound(t *testing.T) { +func TestVector80TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorJSON := `{ "name": "ordered_tool_round", "stage": "turn", @@ -6024,7 +5970,7 @@ func TestVector81TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector82TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { +func TestVector81TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "permission_denial_is_model_visible", "stage": "turn", @@ -6076,7 +6022,7 @@ func TestVector82TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector83WireConformanceToRequestAnthropicImageFormat(t *testing.T) { +func TestVector82WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorJSON := `{ "name": "anthropic_image_format", "description": "§7.5 — Anthropic: ImagePart uses base64 source block with media_type, not image_url format.", @@ -6147,7 +6093,7 @@ func TestVector83WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector84WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { +func TestVector83WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { vectorJSON := `{ "name": "anthropic_max_tokens_required", "description": "§7.5 — Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).", @@ -6203,7 +6149,7 @@ func TestVector84WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector85WireConformanceToRequestAnthropicOptions(t *testing.T) { +func TestVector84WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorJSON := `{ "name": "anthropic_options", "description": "Anthropic: option names map correctly (topK, stopSequences)", @@ -6273,7 +6219,7 @@ func TestVector85WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector86WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { +func TestVector85WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { vectorJSON := `{ "name": "anthropic_system_separate", "description": "§7.5 — Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.", @@ -6341,7 +6287,7 @@ func TestVector86WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector87WireConformanceToRequestAnthropicToolWire(t *testing.T) { +func TestVector86WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorJSON := `{ "name": "anthropic_tool_wire", "description": "Anthropic: tools use input_schema (not parameters), name at top level", @@ -6427,7 +6373,7 @@ func TestVector87WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector88WireConformanceToRequestChatAudioMp3(t *testing.T) { +func TestVector87WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorJSON := `{ "name": "chat_audio_mp3", "description": "§7.1.2 — AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.", @@ -6486,7 +6432,7 @@ func TestVector88WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector89WireConformanceToRequestChatAudioPart(t *testing.T) { +func TestVector88WireConformanceToRequestChatAudioPart(t *testing.T) { vectorJSON := `{ "name": "chat_audio_part", "description": "§7.1.2 — AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.", @@ -6545,7 +6491,7 @@ func TestVector89WireConformanceToRequestChatAudioPart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector90WireConformanceToRequestChatImageBase64(t *testing.T) { +func TestVector89WireConformanceToRequestChatImageBase64(t *testing.T) { vectorJSON := `{ "name": "chat_image_base64", "description": "Base64 data URI image part preserved as-is in image_url", @@ -6610,7 +6556,7 @@ func TestVector90WireConformanceToRequestChatImageBase64(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector91WireConformanceToRequestChatImagePart(t *testing.T) { +func TestVector90WireConformanceToRequestChatImagePart(t *testing.T) { vectorJSON := `{ "name": "chat_image_part", "description": "Image content part converted to OpenAI image_url format", @@ -6675,7 +6621,7 @@ func TestVector91WireConformanceToRequestChatImagePart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector92WireConformanceToRequestChatMultipartContent(t *testing.T) { +func TestVector91WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorJSON := `{ "name": "chat_multipart_content", "description": "§7.1.1/§7.1.2 — Message with TextPart + ImagePart MUST produce an array of typed content blocks.", @@ -6740,7 +6686,7 @@ func TestVector92WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector93WireConformanceToRequestChatSimple(t *testing.T) { +func TestVector92WireConformanceToRequestChatSimple(t *testing.T) { vectorJSON := `{ "name": "chat_simple", "description": "§7.1.7 — Two messages (system + user), no tools, no options. Minimal chat request.", @@ -6803,7 +6749,7 @@ func TestVector93WireConformanceToRequestChatSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector94WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { +func TestVector93WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { vectorJSON := `{ "name": "chat_single_text_optimized", "description": "§7.1.1 — Message with exactly 1 TextPart MUST produce a plain string content, not an array.", @@ -6853,7 +6799,7 @@ func TestVector94WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector95WireConformanceToRequestChatWithOptions(t *testing.T) { +func TestVector94WireConformanceToRequestChatWithOptions(t *testing.T) { vectorJSON := `{ "name": "chat_with_options", "description": "§7.1.5 — All standard ModelOptions mapped to OpenAI parameter names.", @@ -6922,7 +6868,7 @@ func TestVector95WireConformanceToRequestChatWithOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector96WireConformanceToRequestEmbeddingWire(t *testing.T) { +func TestVector95WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorJSON := `{ "name": "embedding_wire", "description": "§7.2 — Embedding request: text extracted from messages, single string input (not array).", @@ -6967,7 +6913,7 @@ func TestVector96WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector97WireConformanceToRequestImageWire(t *testing.T) { +func TestVector96WireConformanceToRequestImageWire(t *testing.T) { vectorJSON := `{ "name": "image_wire", "description": "§7.3 — Image generation request: prompt extracted from last user message.", @@ -7012,7 +6958,7 @@ func TestVector97WireConformanceToRequestImageWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector98WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { +func TestVector97WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorJSON := `{ "name": "kind_to_json_type_mapping", "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.", @@ -7132,7 +7078,7 @@ func TestVector98WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector99WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { +func TestVector98WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { vectorJSON := `{ "name": "options_additional_properties", "description": "§7.1.5 — additionalProperties from ModelOptions MUST be merged into the request as top-level keys.", @@ -7187,7 +7133,7 @@ func TestVector99WireConformanceToRequestOptionsAdditionalProperties(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector100WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { +func TestVector99WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { vectorJSON := `{ "name": "options_max_completion_tokens", "description": "§7.1.5 — maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.", @@ -7240,7 +7186,7 @@ func TestVector100WireConformanceToRequestOptionsMaxCompletionTokens(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector101WireConformanceToRequestOptionsStopSequences(t *testing.T) { +func TestVector100WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorJSON := `{ "name": "options_stop_sequences", "description": "§7.1.5 — stopSequences MUST map to 'stop' in the OpenAI wire format.", @@ -7299,7 +7245,7 @@ func TestVector101WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector102WireConformanceToRequestResponsesSimple(t *testing.T) { +func TestVector101WireConformanceToRequestResponsesSimple(t *testing.T) { vectorJSON := `{ "name": "responses_simple", "description": "Responses API: system becomes instructions, messages become input items", @@ -7359,7 +7305,7 @@ func TestVector102WireConformanceToRequestResponsesSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector103WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { +func TestVector102WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "responses_structured_output", "description": "Responses API: structured output uses text.format.json_schema", @@ -7447,7 +7393,7 @@ func TestVector103WireConformanceToRequestResponsesStructuredOutput(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector104WireConformanceToRequestResponsesWithTools(t *testing.T) { +func TestVector103WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorJSON := `{ "name": "responses_with_tools", "description": "Responses API: tools use flat format (no nested function key)", @@ -7528,7 +7474,7 @@ func TestVector104WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector105WireConformanceToRequestStructuredOutput(t *testing.T) { +func TestVector104WireConformanceToRequestStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "structured_output", "description": "§7.1.6 — outputs converted to response_format with json_schema, strict=true, additionalProperties=false.", @@ -7612,7 +7558,7 @@ func TestVector105WireConformanceToRequestStructuredOutput(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector106WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { +func TestVector105WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { vectorJSON := `{ "name": "structured_output_nested_optional", "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", @@ -7715,7 +7661,7 @@ func TestVector106WireConformanceToRequestStructuredOutputNestedOptional(t *test vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector107WireConformanceToRequestToolsBindingsStripped(t *testing.T) { +func TestVector106WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorJSON := `{ "name": "tools_bindings_stripped", "description": "§7.1.3 — Parameters listed in bindings MUST be stripped from wire tools (properties AND required).", @@ -7808,7 +7754,7 @@ func TestVector107WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector108WireConformanceToRequestToolsFunctionWire(t *testing.T) { +func TestVector107WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorJSON := `{ "name": "tools_function_wire", "description": "§7.1.3 — FunctionTool projected as OpenAI function definition with JSON Schema parameters.", @@ -7891,7 +7837,7 @@ func TestVector108WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector109WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { +func TestVector108WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorJSON := `{ "name": "tools_null_when_empty", "description": "§7.1.3 — When no tools are defined, the tools key MUST be absent from the request entirely.", @@ -7941,7 +7887,7 @@ func TestVector109WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector110WireConformanceToRequestToolsStrictMode(t *testing.T) { +func TestVector109WireConformanceToRequestToolsStrictMode(t *testing.T) { vectorJSON := `{ "name": "tools_strict_mode", "description": "§7.1.3 — FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.", diff --git a/runtime/go/prompty/model/vectorbridge/bridge.go b/runtime/go/prompty/model/vectorbridge/bridge.go index 555bda37a..6140421cb 100644 --- a/runtime/go/prompty/model/vectorbridge/bridge.go +++ b/runtime/go/prompty/model/vectorbridge/bridge.go @@ -56,27 +56,6 @@ func DiscoveryConformanceMapModel(impl fixtures.DiscoveryConformance) func(rawIn } } -// LiveChatConformanceComplete decodes LiveChatConformance.complete vector input and invokes a -// typed fixtures.LiveChatConformance. Adapt the returned func into your Adapter, e.g.: -// -// VectorAdapters["LiveChatConformance.complete"] = bridged(vectorbridge.LiveChatConformanceComplete(impl)) -func LiveChatConformanceComplete(impl fixtures.LiveChatConformance) func(rawInput any) (any, error) { - return func(rawInput any) (any, error) { - payload, _ := rawInput.(map[string]any) - var input interface{} - { - inputBytes, marshalErr := json.Marshal(payload["input"]) - if marshalErr != nil { - return nil, marshalErr - } - if unmarshalErr := json.Unmarshal(inputBytes, &input); unmarshalErr != nil { - return nil, unmarshalErr - } - } - return impl.Complete(input) - } -} - // LoadConformanceLoad decodes LoadConformance.load vector input and invokes a // typed fixtures.LoadConformance. Adapt the returned func into your Adapter, e.g.: // diff --git a/runtime/go/prompty/vectoradapters/vectoradapters.go b/runtime/go/prompty/vectoradapters/vectoradapters.go index 617e31ed7..568487144 100644 --- a/runtime/go/prompty/vectoradapters/vectoradapters.go +++ b/runtime/go/prompty/vectoradapters/vectoradapters.go @@ -79,9 +79,7 @@ var VectorAdapters = map[string]Adapter{ // VectorWaivers records explicit, honest conformance gaps. Empty: every gap is // being driven to green rather than waived. -var VectorWaivers = map[string]string{ - "LiveChatConformance.complete": "drill: consumer-owned transport-seam probe (Python/TS only)", -} +var VectorWaivers = map[string]string{} // VectorDoubles is reserved for deterministic test doubles. var VectorDoubles = map[string]any{} diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LiveChatConformance.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LiveChatConformance.java deleted file mode 100644 index c243071bc..000000000 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LiveChatConformance.java +++ /dev/null @@ -1,12 +0,0 @@ -// -// Code generated by Typra emitter; DO NOT EDIT. -package com.microsoft.prompty.model; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public interface LiveChatConformance { - Object complete(Object input); -} diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java index addcba2ad..1056bd1fb 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java @@ -77,11 +77,7 @@ public static Map adapters() { } public static Map waivers() { - Map waivers = new LinkedHashMap<>(); - waivers.put( - "LiveChatConformance.complete", - "drill: consumer-owned transport-seam probe (Python/TS only)"); - return waivers; + return new LinkedHashMap<>(); } public static Object doubles() { diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java index a1d21a6b0..81e8f9050 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java @@ -169,665 +169,658 @@ private static void vector15DiscoveryConformanceMapModelOpenaiModelFinetuneNoOwn VectorRunner.runVector("DiscoveryConformance", "mapModel", vector, false, seam()); } - private static void vector16LiveChatConformanceCompleteOpenaiChatDrill() throws Exception { - StringBuilder sb = new StringBuilder(); - sb.append("{\"name\":\"openai_chat_drill\",\"stage\":\"drill\",\"provider\":\"openai\",\"input\":{\"provider\":\"openai\",\"model\":\"gpt-4o-mini\",\"apiKey\":{\"$env\":\"OPENAI_API_KEY\"},\"endpoint\":{\"$env\":\"OPENAI_BASE_URL\"},\"messages\":[{\"role\":\"user\",\"content\":\"Say hello in exactly one word.\"}],\"options\":{\"temperature\":0,\"maxOutputTokens\":16}},\"exchange\":{\"transport\":{\"baseUrl\":{\"$env\":\"OPENAI_BASE_URL\"}},\"cassette\":{\"$json\":\"cassettes/openai_chat_drill.json\"},\"planes\":[\"transport\",\"wire\",\"semantic\"]},\"expected\":{\"role\":\"assistant\",\"contentNonEmpty\":true,\"finishReasonInEnum\":true},\"operation\":\"complete\"}"); - Object vector = TypraJson.parse(sb.toString()); - VectorRunner.runVector("LiveChatConformance", "complete", vector, false, seam()); - } - - private static void vector17LoadConformanceLoadBasicLoad() throws Exception { + private static void vector16LoadConformanceLoadBasicLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"basic_load\",\"description\":\"Load basic.prompty with env var resolution and verify all PromptAgent fields\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"basic-prompt\",\"description\":\"A basic prompt for testing\",\"metadata\":{\"authors\":[\"testauthor\"]},\"model\":{\"id\":\"gpt-4\",\"provider\":\"openai\",\"apiType\":\"chat\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"},\"options\":{\"temperature\":0.7,\"maxOutputTokens\":1000}},\"inputs\":[{\"name\":\"firstName\",\"kind\":\"string\",\"default\":\"Jane\"},{\"name\":\"lastName\",\"kind\":\"string\",\"default\":\"Doe\"},{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What is the meaning of life?\"}],\"template\":{\"format\":{\"kind\":\"jinja2\"},\"parser\":{\"kind\":\"prompty\"}},\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector18LoadConformanceLoadConnectionTypesLoad() throws Exception { + private static void vector17LoadConformanceLoadConnectionTypesLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector19LoadConformanceLoadEmbeddingLoad() throws Exception { + private static void vector18LoadConformanceLoadEmbeddingLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"embedding_load\",\"description\":\"Load embedding.prompty and verify apiType=embedding with correct model id\",\"stage\":\"load\",\"input\":{\"fixture\":\"embedding.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"embedding\",\"model\":{\"id\":\"text-embedding-3-small\",\"provider\":\"openai\",\"apiType\":\"embedding\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"}},\"inputs\":[{\"name\":\"text\",\"kind\":\"string\",\"default\":\"Hello world\"}],\"instructions\":\"{{text}}\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector20LoadConformanceLoadEmptyFrontmatterBodyOnly() throws Exception { + private static void vector19LoadConformanceLoadEmptyFrontmatterBodyOnly() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"empty_frontmatter_body_only\",\"description\":\"Empty frontmatter with body \\u2014 instructions extracted, defaults applied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"empty-fm\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"empty-fm\",\"model\":null,\"inputs\":null,\"tools\":null},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector21LoadConformanceLoadEnvDefault() throws Exception { + private static void vector20LoadConformanceLoadEnvDefault() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector22LoadConformanceLoadEnvMissingError() throws Exception { + private static void vector21LoadConformanceLoadEnvMissingError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"env_missing_error\",\"description\":\"${env:NONEXISTENT} with no default and var not set raises an error\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-error-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:NONEXISTENT}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"env_var_not_set\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector23LoadConformanceLoadEnvResolution() throws Exception { + private static void vector22LoadConformanceLoadEnvResolution() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"env_resolution\",\"description\":\"${env:MY_VAR} resolves to the environment variable value when set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MY_VAR}\"}}},\"env\":{\"MY_VAR\":\"hello\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"hello\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector24LoadConformanceLoadFileReferenceInTreeAllowed() throws Exception { + private static void vector23LoadConformanceLoadFileReferenceInTreeAllowed() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"file_reference_in_tree_allowed\",\"description\":\"A ${file:sub/data.json} reference that stays within the prompt file's directory tree resolves to the parsed content (containment control alongside the traversal-reject vector)\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-in-tree-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:sub/data.json}\"}},\"files\":{\"sub/data.json\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\",\"apiKey\":\"ok\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector25LoadConformanceLoadFileReferenceTraversalRejected() throws Exception { + private static void vector24LoadConformanceLoadFileReferenceTraversalRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"file_reference_traversal_rejected\",\"description\":\"A ${file:../secret.json} reference that escapes the prompt file's directory via `..` is rejected even though the target file exists, preventing arbitrary file disclosure (GHSA-7pfc-5v9r-j299). Uses forward-slash separators so the vector holds on every OS.\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-traversal-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:../secret.json}\"}},\"files\":{\"../secret.json\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"leaked\"}}},\"expectedError\":{\"kind\":\"file_reference\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector26LoadConformanceLoadFileResolution() throws Exception { + private static void vector25LoadConformanceLoadFileResolution() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"file_resolution\",\"description\":\"${file:shared_connection.json} resolves to the parsed JSON content of the referenced file\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"file-ref-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:shared_connection.json}\"}},\"env\":{\"OPENAI_API_KEY\":\"shared-key\"},\"files\":{\"shared_connection.json\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\",\"apiKey\":\"shared-key\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector27LoadConformanceLoadImageApitypeLoad() throws Exception { + private static void vector26LoadConformanceLoadImageApitypeLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"image_apitype_load\",\"description\":\"An image-generation prompt loads with apiType image and passthrough additionalProperties model options\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}}},\"expected\":{\"kind\":\"prompt\",\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector28LoadConformanceLoadInputScalarShorthand() throws Exception { + private static void vector27LoadConformanceLoadInputScalarShorthand() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"input_scalar_shorthand\",\"description\":\"Scalar input values create typed Property with default set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"scalar-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]}},\"expected\":{\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector29LoadConformanceLoadInputValidationDefaultFill() throws Exception { + private static void vector28LoadConformanceLoadInputValidationDefaultFill() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"input_validation_default_fill\",\"description\":\"An input with a default value is filled in when no value is provided\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"default-fill-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"name\",\"kind\":\"string\",\"default\":\"world\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{\"name\":\"world\"}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector30LoadConformanceLoadInputValidationExampleNotUsed() throws Exception { + private static void vector29LoadConformanceLoadInputValidationExampleNotUsed() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"input_validation_example_not_used\",\"description\":\"An input with example='sample' but no default must NOT use the example at runtime\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"example-not-used-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"required\":false,\"example\":\"sample\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector31LoadConformanceLoadInputValidationOptionalOmit() throws Exception { + private static void vector30LoadConformanceLoadInputValidationOptionalOmit() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"input_validation_optional_omit\",\"description\":\"A non-required input with no default and no value is omitted (not an error)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"optional-omit-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"nickname\",\"kind\":\"string\",\"required\":false}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector32LoadConformanceLoadInputValidationRequired() throws Exception { + private static void vector31LoadConformanceLoadInputValidationRequired() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"input_validation_required\",\"description\":\"A required input with no default and no value provided raises ValueError\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"required-input-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},\"inputs\":{}},\"expectedError\":{\"kind\":\"missing_required_input\",\"field\":\"city\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector33LoadConformanceLoadInstructionsFromBody() throws Exception { + private static void vector32LoadConformanceLoadInstructionsFromBody() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"instructions_from_body\",\"description\":\"The markdown body after the closing --- becomes the instructions field\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector34LoadConformanceLoadInvalidFrontmatterError() throws Exception { + private static void vector33LoadConformanceLoadInvalidFrontmatterError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"invalid_frontmatter_error\",\"description\":\"Malformed YAML in frontmatter raises a parse error\",\"stage\":\"load\",\"input\":{\"frontmatter_raw\":\"---\\nname: [invalid\\n---\\nHello\"},\"expectedError\":{\"kind\":\"invalid_frontmatter\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector35LoadConformanceLoadKindAlwaysPrompt() throws Exception { + private static void vector34LoadConformanceLoadKindAlwaysPrompt() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"kind_always_prompt\",\"description\":\"The loader always injects kind='prompt' \\u2014 .prompty files never specify kind themselves\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector36LoadConformanceLoadMinimalLoad() throws Exception { + private static void vector35LoadConformanceLoadMinimalLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"minimal_load\",\"description\":\"Load minimal.prompty with model shorthand and bare-minimum frontmatter\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"minimal\",\"model\":{\"id\":\"gpt-4\"},\"instructions\":\"system:\\nHello world.\",\"inputs\":null,\"outputs\":null,\"tools\":null},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector37LoadConformanceLoadMissingFileError() throws Exception { + private static void vector36LoadConformanceLoadMissingFileError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"missing_file_error\",\"description\":\"Loading a nonexistent .prompty file raises FileNotFoundError\",\"stage\":\"load\",\"input\":{\"fixture\":\"nonexistent.prompty\"},\"expectedError\":{\"kind\":\"file_not_found\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector38LoadConformanceLoadModelShorthand() throws Exception { + private static void vector37LoadConformanceLoadModelShorthand() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"model_shorthand\",\"description\":\"String shorthand 'model: gpt-4o' expands to a Model object with id set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"test\",\"model\":\"gpt-4o\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"test\",\"model\":{\"id\":\"gpt-4o\"}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector39LoadConformanceLoadStructuredOutputsLoad() throws Exception { + private static void vector38LoadConformanceLoadStructuredOutputsLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_outputs_load\",\"description\":\"Outputs load as a list of properties preserving kind, description, and required flags\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"structured-output\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]}},\"expected\":{\"kind\":\"prompt\",\"name\":\"structured-output\",\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector40LoadConformanceLoadTemplateStringInvalid() throws Exception { + private static void vector39LoadConformanceLoadTemplateStringInvalid() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"template_string_invalid\",\"description\":\"Template as a bare string is not valid v2 \\u2014 must be an object with format/parser\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"shorthand-test\",\"model\":{\"id\":\"gpt-4\"},\"template\":\"jinja2\"}},\"expectedError\":{\"kind\":\"invalid_template\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector41LoadConformanceLoadToolsCustomLoad() throws Exception { + private static void vector40LoadConformanceLoadToolsCustomLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_custom_load\",\"description\":\"Load a prompty with an unknown tool kind \\u2014 falls through to CustomTool\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"custom-tool-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://custom.example.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\"}]},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector42LoadConformanceLoadToolsFunctionLoad() throws Exception { + private static void vector41LoadConformanceLoadToolsFunctionLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_function_load\",\"description\":\"Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings\",\"stage\":\"load\",\"input\":{\"fixture\":\"tools_function.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"function-tools\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather in a given location\",\"strict\":true,\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"description\":\"The city and state, e.g. San Francisco, CA\"},{\"name\":\"unit\",\"kind\":\"string\",\"description\":\"Temperature unit\",\"enumValues\":[\"celsius\",\"fahrenheit\"]}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"inputs\":[{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What's the weather in Paris?\"},{\"name\":\"preferred_unit\",\"kind\":\"string\",\"default\":\"celsius\"}],\"instructions\":\"system:\\nYou are a helpful assistant with access to tools.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector43LoadConformanceLoadToolsMcpLoad() throws Exception { + private static void vector42LoadConformanceLoadToolsMcpLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_mcp_load\",\"description\":\"Load a prompty with an MCP tool \\u2014 kind, serverName, connection preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"mcp-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-mcp\"}}]}},\"expected\":{\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\"}]},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector44LoadConformanceLoadToolsOpenapiLoad() throws Exception { + private static void vector43LoadConformanceLoadToolsOpenapiLoad() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_openapi_load\",\"description\":\"Load a prompty with an OpenAPI tool \\u2014 specification path preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"openapi-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://api.weather.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\"}]},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector45TurnConformanceReplayMaxIterations() throws Exception { + private static void vector44TurnConformanceReplayMaxIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector46TurnConformanceReplayNoTool() throws Exception { + private static void vector45TurnConformanceReplayNoTool() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector47TurnConformanceReplayPermissionDenied() throws Exception { + private static void vector46TurnConformanceReplayPermissionDenied() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector48TurnConformanceReplayToolFailure() throws Exception { + private static void vector47TurnConformanceReplayToolFailure() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector49TurnConformanceReplayToolSuccess() throws Exception { + private static void vector48TurnConformanceReplayToolSuccess() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector50TurnConformanceRunAssistantToolCallsMetadata() throws Exception { + private static void vector49TurnConformanceRunAssistantToolCallsMetadata() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector51TurnConformanceRunAsyncToolFunction() throws Exception { + private static void vector50TurnConformanceRunAsyncToolFunction() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector52TurnConformanceRunBindingsInjected() throws Exception { + private static void vector51TurnConformanceRunBindingsInjected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\\u00b0C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector53TurnConformanceRunCancellationBeforeLlm() throws Exception { + private static void vector52TurnConformanceRunCancellationBeforeLlm() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_before_llm\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector54TurnConformanceRunCancellationBetweenIterations() throws Exception { + private static void vector53TurnConformanceRunCancellationBetweenIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_iterations\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector55TurnConformanceRunCancellationBetweenTools() throws Exception { + private static void vector54TurnConformanceRunCancellationBetweenTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_tools\",\"description\":\"\\u00a713.2 Cancellation \\u2014 LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector56TurnConformanceRunContextNoTrimWhenFits() throws Exception { + private static void vector55TurnConformanceRunContextNoTrimWhenFits() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector57TurnConformanceRunContextPreservesSystemMessages() throws Exception { + private static void vector56TurnConformanceRunContextPreservesSystemMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_preserves_system_messages\",\"description\":\"\\u00a713.3 Context Window \\u2014 Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \\u2014 they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector58TurnConformanceRunContextTrimBasic() throws Exception { + private static void vector57TurnConformanceRunContextTrimBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_trim_basic\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector59TurnConformanceRunEmptyToolResult() throws Exception { + private static void vector58TurnConformanceRunEmptyToolResult() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \\u2014 it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector60TurnConformanceRunEventsBasicToolLoop() throws Exception { + private static void vector59TurnConformanceRunEventsBasicToolLoop() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_basic_tool_loop\",\"description\":\"\\u00a713.1 Events \\u2014 A 2-turn tool-call loop emits the full event lifecycle: status \\u2192 tool_call_start \\u2192 tool_result \\u2192 messages_updated \\u2192 done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\\u00b0F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector61TurnConformanceRunEventsErrorLogged() throws Exception { + private static void vector60TurnConformanceRunEventsErrorLogged() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_error_logged\",\"description\":\"\\u00a713.1 Events \\u0097 A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \\u0097 this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector62TurnConformanceRunEventsNoTools() throws Exception { + private static void vector61TurnConformanceRunEventsNoTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_no_tools\",\"description\":\"\\u00a713.1 Events \\u2014 Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector63TurnConformanceRunGuardrailAllPass() throws Exception { + private static void vector62TurnConformanceRunGuardrailAllPass() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_all_pass\",\"description\":\"\\u00a713.4 Guardrails \\u2014 All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector64TurnConformanceRunGuardrailInputDeny() throws Exception { + private static void vector63TurnConformanceRunGuardrailInputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_input_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector65TurnConformanceRunGuardrailOutputDeny() throws Exception { + private static void vector64TurnConformanceRunGuardrailOutputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_output_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector66TurnConformanceRunGuardrailToolDeny() throws Exception { + private static void vector65TurnConformanceRunGuardrailToolDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_tool_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \\u2014 its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector67TurnConformanceRunMaxIterationsExceeded() throws Exception { + private static void vector66TurnConformanceRunMaxIterationsExceeded() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\\u00b0F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\\u00b0F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\\u00b0F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\\u00b0F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\\u00b0F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\\u00b0F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\\u00b0F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\\u00b0F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\\u00b0F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector68TurnConformanceRunMultiTurnToolCalls() throws Exception { + private static void vector67TurnConformanceRunMultiTurnToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\\u00b0C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector69TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { + private static void vector68TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \\u2014 get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\\u00b0F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector70TurnConformanceRunNoToolCalls() throws Exception { + private static void vector69TurnConformanceRunNoToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \\u2014 the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector71TurnConformanceRunParallelToolsBasic() throws Exception { + private static void vector70TurnConformanceRunParallelToolsBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_basic\",\"description\":\"\\u00a713.6 Parallel Tools \\u2014 LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector72TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { + private static void vector71TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\\u00a713.6 Parallel Tools + \\u00a713.4 Guardrails \\u2014 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector73TurnConformanceRunSingleToolCall() throws Exception { + private static void vector72TurnConformanceRunSingleToolCall() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector74TurnConformanceRunSteeringInjectMessage() throws Exception { + private static void vector73TurnConformanceRunSteeringInjectMessage() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_inject_message\",\"description\":\"\\u00a713.5 Steering \\u2014 After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector75TurnConformanceRunSteeringMultipleMessages() throws Exception { + private static void vector74TurnConformanceRunSteeringMultipleMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_multiple_messages\",\"description\":\"\\u00a713.5 Steering \\u2014 Two steering messages are queued and both injected before iteration 2. Tests atomic drain \\u2014 both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\\u00b0F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector76TurnConformanceRunToolNotRegisteredError() throws Exception { + private static void vector75TurnConformanceRunToolNotRegisteredError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector77TurnConformanceRunToolResultMessageFormat() throws Exception { + private static void vector76TurnConformanceRunToolResultMessageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\\u00b0F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\\u00b0F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector78TurnConformanceRunTurnCancelBeforeContext() throws Exception { + private static void vector77TurnConformanceRunTurnCancelBeforeContext() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector79TurnConformanceRunTurnDelegatedProviderState() throws Exception { + private static void vector78TurnConformanceRunTurnDelegatedProviderState() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector80TurnConformanceRunTurnFinalOutput() throws Exception { + private static void vector79TurnConformanceRunTurnFinalOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector81TurnConformanceRunTurnOrderedToolRound() throws Exception { + private static void vector80TurnConformanceRunTurnOrderedToolRound() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector82TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { + private static void vector81TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector83WireConformanceToRequestAnthropicImageFormat() throws Exception { + private static void vector82WireConformanceToRequestAnthropicImageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_image_format\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector84WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { + private static void vector83WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector85WireConformanceToRequestAnthropicOptions() throws Exception { + private static void vector84WireConformanceToRequestAnthropicOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector86WireConformanceToRequestAnthropicSystemSeparate() throws Exception { + private static void vector85WireConformanceToRequestAnthropicSystemSeparate() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_system_separate\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector87WireConformanceToRequestAnthropicToolWire() throws Exception { + private static void vector86WireConformanceToRequestAnthropicToolWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector88WireConformanceToRequestChatAudioMp3() throws Exception { + private static void vector87WireConformanceToRequestChatAudioMp3() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_mp3\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector89WireConformanceToRequestChatAudioPart() throws Exception { + private static void vector88WireConformanceToRequestChatAudioPart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_part\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector90WireConformanceToRequestChatImageBase64() throws Exception { + private static void vector89WireConformanceToRequestChatImageBase64() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector91WireConformanceToRequestChatImagePart() throws Exception { + private static void vector90WireConformanceToRequestChatImagePart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector92WireConformanceToRequestChatMultipartContent() throws Exception { + private static void vector91WireConformanceToRequestChatMultipartContent() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_multipart_content\",\"description\":\"\\u00a77.1.1/\\u00a77.1.2 \\u2014 Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector93WireConformanceToRequestChatSimple() throws Exception { + private static void vector92WireConformanceToRequestChatSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_simple\",\"description\":\"\\u00a77.1.7 \\u2014 Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector94WireConformanceToRequestChatSingleTextOptimized() throws Exception { + private static void vector93WireConformanceToRequestChatSingleTextOptimized() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_single_text_optimized\",\"description\":\"\\u00a77.1.1 \\u2014 Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector95WireConformanceToRequestChatWithOptions() throws Exception { + private static void vector94WireConformanceToRequestChatWithOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_with_options\",\"description\":\"\\u00a77.1.5 \\u2014 All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector96WireConformanceToRequestEmbeddingWire() throws Exception { + private static void vector95WireConformanceToRequestEmbeddingWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"embedding_wire\",\"description\":\"\\u00a77.2 \\u2014 Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector97WireConformanceToRequestImageWire() throws Exception { + private static void vector96WireConformanceToRequestImageWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"image_wire\",\"description\":\"\\u00a77.3 \\u2014 Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector98WireConformanceToRequestKindToJsonTypeMapping() throws Exception { + private static void vector97WireConformanceToRequestKindToJsonTypeMapping() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\\u00a77.1.4 \\u2014 All Property kind values mapped to JSON Schema types: string\\u2192string, integer\\u2192integer, float\\u2192number, boolean\\u2192boolean, array\\u2192array, object\\u2192object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector99WireConformanceToRequestOptionsAdditionalProperties() throws Exception { + private static void vector98WireConformanceToRequestOptionsAdditionalProperties() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_additional_properties\",\"description\":\"\\u00a77.1.5 \\u2014 additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector100WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { + private static void vector99WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_max_completion_tokens\",\"description\":\"\\u00a77.1.5 \\u2014 maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector101WireConformanceToRequestOptionsStopSequences() throws Exception { + private static void vector100WireConformanceToRequestOptionsStopSequences() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_stop_sequences\",\"description\":\"\\u00a77.1.5 \\u2014 stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector102WireConformanceToRequestResponsesSimple() throws Exception { + private static void vector101WireConformanceToRequestResponsesSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector103WireConformanceToRequestResponsesStructuredOutput() throws Exception { + private static void vector102WireConformanceToRequestResponsesStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector104WireConformanceToRequestResponsesWithTools() throws Exception { + private static void vector103WireConformanceToRequestResponsesWithTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector105WireConformanceToRequestStructuredOutput() throws Exception { + private static void vector104WireConformanceToRequestStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output\",\"description\":\"\\u00a77.1.6 \\u2014 outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector106WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { + private static void vector105WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output_nested_optional\",\"description\":\"\\u00a77.1.4/\\u00a77.1.6 \\u2014 OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector107WireConformanceToRequestToolsBindingsStripped() throws Exception { + private static void vector106WireConformanceToRequestToolsBindingsStripped() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_bindings_stripped\",\"description\":\"\\u00a77.1.3 \\u2014 Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector108WireConformanceToRequestToolsFunctionWire() throws Exception { + private static void vector107WireConformanceToRequestToolsFunctionWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_function_wire\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector109WireConformanceToRequestToolsNullWhenEmpty() throws Exception { + private static void vector108WireConformanceToRequestToolsNullWhenEmpty() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_null_when_empty\",\"description\":\"\\u00a77.1.3 \\u2014 When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector110WireConformanceToRequestToolsStrictMode() throws Exception { + private static void vector109WireConformanceToRequestToolsStrictMode() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_strict_mode\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); @@ -852,101 +845,100 @@ public static void run() { failed += runCase("DiscoveryConformance.mapModel:foundry_deployment_nested_arm", VectorConformanceTests::vector13DiscoveryConformanceMapModelFoundryDeploymentNestedArm); failed += runCase("DiscoveryConformance.mapModel:openai_model_basic", VectorConformanceTests::vector14DiscoveryConformanceMapModelOpenaiModelBasic); failed += runCase("DiscoveryConformance.mapModel:openai_model_finetune_no_owner", VectorConformanceTests::vector15DiscoveryConformanceMapModelOpenaiModelFinetuneNoOwner); - failed += runCase("LiveChatConformance.complete:openai_chat_drill", VectorConformanceTests::vector16LiveChatConformanceCompleteOpenaiChatDrill); - failed += runCase("LoadConformance.load:basic_load", VectorConformanceTests::vector17LoadConformanceLoadBasicLoad); - failed += runCase("LoadConformance.load:connection_types_load", VectorConformanceTests::vector18LoadConformanceLoadConnectionTypesLoad); - failed += runCase("LoadConformance.load:embedding_load", VectorConformanceTests::vector19LoadConformanceLoadEmbeddingLoad); - failed += runCase("LoadConformance.load:empty_frontmatter_body_only", VectorConformanceTests::vector20LoadConformanceLoadEmptyFrontmatterBodyOnly); - failed += runCase("LoadConformance.load:env_default", VectorConformanceTests::vector21LoadConformanceLoadEnvDefault); - failed += runCase("LoadConformance.load:env_missing_error", VectorConformanceTests::vector22LoadConformanceLoadEnvMissingError); - failed += runCase("LoadConformance.load:env_resolution", VectorConformanceTests::vector23LoadConformanceLoadEnvResolution); - failed += runCase("LoadConformance.load:file_reference_in_tree_allowed", VectorConformanceTests::vector24LoadConformanceLoadFileReferenceInTreeAllowed); - failed += runCase("LoadConformance.load:file_reference_traversal_rejected", VectorConformanceTests::vector25LoadConformanceLoadFileReferenceTraversalRejected); - failed += runCase("LoadConformance.load:file_resolution", VectorConformanceTests::vector26LoadConformanceLoadFileResolution); - failed += runCase("LoadConformance.load:image_apitype_load", VectorConformanceTests::vector27LoadConformanceLoadImageApitypeLoad); - failed += runCase("LoadConformance.load:input_scalar_shorthand", VectorConformanceTests::vector28LoadConformanceLoadInputScalarShorthand); - failed += runCase("LoadConformance.load:input_validation_default_fill", VectorConformanceTests::vector29LoadConformanceLoadInputValidationDefaultFill); - failed += runCase("LoadConformance.load:input_validation_example_not_used", VectorConformanceTests::vector30LoadConformanceLoadInputValidationExampleNotUsed); - failed += runCase("LoadConformance.load:input_validation_optional_omit", VectorConformanceTests::vector31LoadConformanceLoadInputValidationOptionalOmit); - failed += runCase("LoadConformance.load:input_validation_required", VectorConformanceTests::vector32LoadConformanceLoadInputValidationRequired); - failed += runCase("LoadConformance.load:instructions_from_body", VectorConformanceTests::vector33LoadConformanceLoadInstructionsFromBody); - failed += runCase("LoadConformance.load:invalid_frontmatter_error", VectorConformanceTests::vector34LoadConformanceLoadInvalidFrontmatterError); - failed += runCase("LoadConformance.load:kind_always_prompt", VectorConformanceTests::vector35LoadConformanceLoadKindAlwaysPrompt); - failed += runCase("LoadConformance.load:minimal_load", VectorConformanceTests::vector36LoadConformanceLoadMinimalLoad); - failed += runCase("LoadConformance.load:missing_file_error", VectorConformanceTests::vector37LoadConformanceLoadMissingFileError); - failed += runCase("LoadConformance.load:model_shorthand", VectorConformanceTests::vector38LoadConformanceLoadModelShorthand); - failed += runCase("LoadConformance.load:structured_outputs_load", VectorConformanceTests::vector39LoadConformanceLoadStructuredOutputsLoad); - failed += runCase("LoadConformance.load:template_string_invalid", VectorConformanceTests::vector40LoadConformanceLoadTemplateStringInvalid); - failed += runCase("LoadConformance.load:tools_custom_load", VectorConformanceTests::vector41LoadConformanceLoadToolsCustomLoad); - failed += runCase("LoadConformance.load:tools_function_load", VectorConformanceTests::vector42LoadConformanceLoadToolsFunctionLoad); - failed += runCase("LoadConformance.load:tools_mcp_load", VectorConformanceTests::vector43LoadConformanceLoadToolsMcpLoad); - failed += runCase("LoadConformance.load:tools_openapi_load", VectorConformanceTests::vector44LoadConformanceLoadToolsOpenapiLoad); - failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector45TurnConformanceReplayMaxIterations); - failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector46TurnConformanceReplayNoTool); - failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector47TurnConformanceReplayPermissionDenied); - failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector48TurnConformanceReplayToolFailure); - failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector49TurnConformanceReplayToolSuccess); - failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector50TurnConformanceRunAssistantToolCallsMetadata); - failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector51TurnConformanceRunAsyncToolFunction); - failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector52TurnConformanceRunBindingsInjected); - failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector53TurnConformanceRunCancellationBeforeLlm); - failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector54TurnConformanceRunCancellationBetweenIterations); - failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector55TurnConformanceRunCancellationBetweenTools); - failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector56TurnConformanceRunContextNoTrimWhenFits); - failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector57TurnConformanceRunContextPreservesSystemMessages); - failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector58TurnConformanceRunContextTrimBasic); - failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector59TurnConformanceRunEmptyToolResult); - failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector60TurnConformanceRunEventsBasicToolLoop); - failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector61TurnConformanceRunEventsErrorLogged); - failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector62TurnConformanceRunEventsNoTools); - failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector63TurnConformanceRunGuardrailAllPass); - failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector64TurnConformanceRunGuardrailInputDeny); - failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector65TurnConformanceRunGuardrailOutputDeny); - failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector66TurnConformanceRunGuardrailToolDeny); - failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector67TurnConformanceRunMaxIterationsExceeded); - failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector68TurnConformanceRunMultiTurnToolCalls); - failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector69TurnConformanceRunMultipleToolCallsSingleTurn); - failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector70TurnConformanceRunNoToolCalls); - failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector71TurnConformanceRunParallelToolsBasic); - failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector72TurnConformanceRunParallelToolsWithGuardrailDeny); - failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector73TurnConformanceRunSingleToolCall); - failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector74TurnConformanceRunSteeringInjectMessage); - failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector75TurnConformanceRunSteeringMultipleMessages); - failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector76TurnConformanceRunToolNotRegisteredError); - failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector77TurnConformanceRunToolResultMessageFormat); - failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector78TurnConformanceRunTurnCancelBeforeContext); - failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector79TurnConformanceRunTurnDelegatedProviderState); - failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector80TurnConformanceRunTurnFinalOutput); - failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector81TurnConformanceRunTurnOrderedToolRound); - failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector82TurnConformanceRunTurnPermissionDenialIsModelVisible); - failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector83WireConformanceToRequestAnthropicImageFormat); - failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector84WireConformanceToRequestAnthropicMaxTokensRequired); - failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector85WireConformanceToRequestAnthropicOptions); - failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector86WireConformanceToRequestAnthropicSystemSeparate); - failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector87WireConformanceToRequestAnthropicToolWire); - failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector88WireConformanceToRequestChatAudioMp3); - failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector89WireConformanceToRequestChatAudioPart); - failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector90WireConformanceToRequestChatImageBase64); - failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector91WireConformanceToRequestChatImagePart); - failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector92WireConformanceToRequestChatMultipartContent); - failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector93WireConformanceToRequestChatSimple); - failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector94WireConformanceToRequestChatSingleTextOptimized); - failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector95WireConformanceToRequestChatWithOptions); - failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector96WireConformanceToRequestEmbeddingWire); - failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector97WireConformanceToRequestImageWire); - failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector98WireConformanceToRequestKindToJsonTypeMapping); - failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector99WireConformanceToRequestOptionsAdditionalProperties); - failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector100WireConformanceToRequestOptionsMaxCompletionTokens); - failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector101WireConformanceToRequestOptionsStopSequences); - failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector102WireConformanceToRequestResponsesSimple); - failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector103WireConformanceToRequestResponsesStructuredOutput); - failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector104WireConformanceToRequestResponsesWithTools); - failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector105WireConformanceToRequestStructuredOutput); - failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector106WireConformanceToRequestStructuredOutputNestedOptional); - failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector107WireConformanceToRequestToolsBindingsStripped); - failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector108WireConformanceToRequestToolsFunctionWire); - failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector109WireConformanceToRequestToolsNullWhenEmpty); - failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector110WireConformanceToRequestToolsStrictMode); + failed += runCase("LoadConformance.load:basic_load", VectorConformanceTests::vector16LoadConformanceLoadBasicLoad); + failed += runCase("LoadConformance.load:connection_types_load", VectorConformanceTests::vector17LoadConformanceLoadConnectionTypesLoad); + failed += runCase("LoadConformance.load:embedding_load", VectorConformanceTests::vector18LoadConformanceLoadEmbeddingLoad); + failed += runCase("LoadConformance.load:empty_frontmatter_body_only", VectorConformanceTests::vector19LoadConformanceLoadEmptyFrontmatterBodyOnly); + failed += runCase("LoadConformance.load:env_default", VectorConformanceTests::vector20LoadConformanceLoadEnvDefault); + failed += runCase("LoadConformance.load:env_missing_error", VectorConformanceTests::vector21LoadConformanceLoadEnvMissingError); + failed += runCase("LoadConformance.load:env_resolution", VectorConformanceTests::vector22LoadConformanceLoadEnvResolution); + failed += runCase("LoadConformance.load:file_reference_in_tree_allowed", VectorConformanceTests::vector23LoadConformanceLoadFileReferenceInTreeAllowed); + failed += runCase("LoadConformance.load:file_reference_traversal_rejected", VectorConformanceTests::vector24LoadConformanceLoadFileReferenceTraversalRejected); + failed += runCase("LoadConformance.load:file_resolution", VectorConformanceTests::vector25LoadConformanceLoadFileResolution); + failed += runCase("LoadConformance.load:image_apitype_load", VectorConformanceTests::vector26LoadConformanceLoadImageApitypeLoad); + failed += runCase("LoadConformance.load:input_scalar_shorthand", VectorConformanceTests::vector27LoadConformanceLoadInputScalarShorthand); + failed += runCase("LoadConformance.load:input_validation_default_fill", VectorConformanceTests::vector28LoadConformanceLoadInputValidationDefaultFill); + failed += runCase("LoadConformance.load:input_validation_example_not_used", VectorConformanceTests::vector29LoadConformanceLoadInputValidationExampleNotUsed); + failed += runCase("LoadConformance.load:input_validation_optional_omit", VectorConformanceTests::vector30LoadConformanceLoadInputValidationOptionalOmit); + failed += runCase("LoadConformance.load:input_validation_required", VectorConformanceTests::vector31LoadConformanceLoadInputValidationRequired); + failed += runCase("LoadConformance.load:instructions_from_body", VectorConformanceTests::vector32LoadConformanceLoadInstructionsFromBody); + failed += runCase("LoadConformance.load:invalid_frontmatter_error", VectorConformanceTests::vector33LoadConformanceLoadInvalidFrontmatterError); + failed += runCase("LoadConformance.load:kind_always_prompt", VectorConformanceTests::vector34LoadConformanceLoadKindAlwaysPrompt); + failed += runCase("LoadConformance.load:minimal_load", VectorConformanceTests::vector35LoadConformanceLoadMinimalLoad); + failed += runCase("LoadConformance.load:missing_file_error", VectorConformanceTests::vector36LoadConformanceLoadMissingFileError); + failed += runCase("LoadConformance.load:model_shorthand", VectorConformanceTests::vector37LoadConformanceLoadModelShorthand); + failed += runCase("LoadConformance.load:structured_outputs_load", VectorConformanceTests::vector38LoadConformanceLoadStructuredOutputsLoad); + failed += runCase("LoadConformance.load:template_string_invalid", VectorConformanceTests::vector39LoadConformanceLoadTemplateStringInvalid); + failed += runCase("LoadConformance.load:tools_custom_load", VectorConformanceTests::vector40LoadConformanceLoadToolsCustomLoad); + failed += runCase("LoadConformance.load:tools_function_load", VectorConformanceTests::vector41LoadConformanceLoadToolsFunctionLoad); + failed += runCase("LoadConformance.load:tools_mcp_load", VectorConformanceTests::vector42LoadConformanceLoadToolsMcpLoad); + failed += runCase("LoadConformance.load:tools_openapi_load", VectorConformanceTests::vector43LoadConformanceLoadToolsOpenapiLoad); + failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector44TurnConformanceReplayMaxIterations); + failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector45TurnConformanceReplayNoTool); + failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector46TurnConformanceReplayPermissionDenied); + failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector47TurnConformanceReplayToolFailure); + failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector48TurnConformanceReplayToolSuccess); + failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector49TurnConformanceRunAssistantToolCallsMetadata); + failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector50TurnConformanceRunAsyncToolFunction); + failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector51TurnConformanceRunBindingsInjected); + failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector52TurnConformanceRunCancellationBeforeLlm); + failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector53TurnConformanceRunCancellationBetweenIterations); + failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector54TurnConformanceRunCancellationBetweenTools); + failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector55TurnConformanceRunContextNoTrimWhenFits); + failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector56TurnConformanceRunContextPreservesSystemMessages); + failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector57TurnConformanceRunContextTrimBasic); + failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector58TurnConformanceRunEmptyToolResult); + failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector59TurnConformanceRunEventsBasicToolLoop); + failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector60TurnConformanceRunEventsErrorLogged); + failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector61TurnConformanceRunEventsNoTools); + failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector62TurnConformanceRunGuardrailAllPass); + failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector63TurnConformanceRunGuardrailInputDeny); + failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector64TurnConformanceRunGuardrailOutputDeny); + failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector65TurnConformanceRunGuardrailToolDeny); + failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector66TurnConformanceRunMaxIterationsExceeded); + failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector67TurnConformanceRunMultiTurnToolCalls); + failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector68TurnConformanceRunMultipleToolCallsSingleTurn); + failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector69TurnConformanceRunNoToolCalls); + failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector70TurnConformanceRunParallelToolsBasic); + failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector71TurnConformanceRunParallelToolsWithGuardrailDeny); + failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector72TurnConformanceRunSingleToolCall); + failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector73TurnConformanceRunSteeringInjectMessage); + failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector74TurnConformanceRunSteeringMultipleMessages); + failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector75TurnConformanceRunToolNotRegisteredError); + failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector76TurnConformanceRunToolResultMessageFormat); + failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector77TurnConformanceRunTurnCancelBeforeContext); + failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector78TurnConformanceRunTurnDelegatedProviderState); + failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector79TurnConformanceRunTurnFinalOutput); + failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector80TurnConformanceRunTurnOrderedToolRound); + failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector81TurnConformanceRunTurnPermissionDenialIsModelVisible); + failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector82WireConformanceToRequestAnthropicImageFormat); + failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector83WireConformanceToRequestAnthropicMaxTokensRequired); + failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector84WireConformanceToRequestAnthropicOptions); + failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector85WireConformanceToRequestAnthropicSystemSeparate); + failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector86WireConformanceToRequestAnthropicToolWire); + failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector87WireConformanceToRequestChatAudioMp3); + failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector88WireConformanceToRequestChatAudioPart); + failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector89WireConformanceToRequestChatImageBase64); + failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector90WireConformanceToRequestChatImagePart); + failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector91WireConformanceToRequestChatMultipartContent); + failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector92WireConformanceToRequestChatSimple); + failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector93WireConformanceToRequestChatSingleTextOptimized); + failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector94WireConformanceToRequestChatWithOptions); + failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector95WireConformanceToRequestEmbeddingWire); + failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector96WireConformanceToRequestImageWire); + failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector97WireConformanceToRequestKindToJsonTypeMapping); + failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector98WireConformanceToRequestOptionsAdditionalProperties); + failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector99WireConformanceToRequestOptionsMaxCompletionTokens); + failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector100WireConformanceToRequestOptionsStopSequences); + failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector101WireConformanceToRequestResponsesSimple); + failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector102WireConformanceToRequestResponsesStructuredOutput); + failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector103WireConformanceToRequestResponsesWithTools); + failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector104WireConformanceToRequestStructuredOutput); + failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector105WireConformanceToRequestStructuredOutputNestedOptional); + failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector106WireConformanceToRequestToolsBindingsStripped); + failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector107WireConformanceToRequestToolsFunctionWire); + failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector108WireConformanceToRequestToolsNullWhenEmpty); + failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector109WireConformanceToRequestToolsStrictMode); if (failed > 0) { throw new AssertionError(failed + " @vector conformance failure(s)"); } diff --git a/runtime/python/prompty/prompty/model/_LiveChatConformance.py b/runtime/python/prompty/prompty/model/_LiveChatConformance.py deleted file mode 100644 index 6ed253b6f..000000000 --- a/runtime/python/prompty/prompty/model/_LiveChatConformance.py +++ /dev/null @@ -1,28 +0,0 @@ -# -########################################## -# WARNING: This is an auto-generated file. -# DO NOT EDIT THIS FILE DIRECTLY -# ANY EDITS WILL BE LOST -########################################## - -from typing import Any, Protocol, runtime_checkable - - -@runtime_checkable -class LiveChatConformance(Protocol): - """Conformance-only seam hosting the "service drill" vectors: a conformance - vector that crosses the REAL Executor->Processor transport seam instead of - stopping at the SDK edge. A drill runs against a live provider when - credentials are present and REPLAYS a recorded cassette when they are not; - the live<->replay parity is the gate. - - Like every other conformance seam this interface exists purely to carry - """ - - def complete(self, input: Any) -> Any: - """Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes""" - raise NotImplementedError - - async def complete_async(self, input: Any) -> Any: - """Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes (async variant)""" - raise NotImplementedError diff --git a/runtime/python/prompty/prompty/model/__init__.py b/runtime/python/prompty/prompty/model/__init__.py index e13bc5b02..a82eda84c 100644 --- a/runtime/python/prompty/prompty/model/__init__.py +++ b/runtime/python/prompty/prompty/model/__init__.py @@ -16,7 +16,6 @@ from ._EventSink import EventSink from ._Executor import Executor from ._HostToolExecutor import HostToolExecutor -from ._LiveChatConformance import LiveChatConformance from ._LoadConformance import LoadConformance from ._ModelLister import ModelLister from ._Parser import Parser @@ -380,7 +379,6 @@ "EventSink", "Executor", "HostToolExecutor", - "LiveChatConformance", "LoadConformance", "ModelLister", "Parser", diff --git a/runtime/python/prompty/tests/model/_drill_replay.py b/runtime/python/prompty/tests/model/_drill_replay.py deleted file mode 100644 index 8075c4016..000000000 --- a/runtime/python/prompty/tests/model/_drill_replay.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Local cassette-replay HTTP transport for the service-drill vector. - -The drill binds the Executor->Processor transport seam via **base-URL redirect**: -the provider SDK is pointed at a local ``http.server`` thread that serves a -recorded response and records the inbound request. This is the consumer-owned -"roster" half of the drill -- deliberately NOT modeled in TypeSpec/Typra, since -cassette bytes and provider-SDK transport binding stay runtime-owned. - -Two use sites share one server: - -* **capture** (tooling, `record_cassette.py`): serve a synthesized provider - response, drive the real executor once, and freeze the captured request as the - cassette's ``request`` plane. -* **replay** (`vector_adapters.py` drill adapter): serve the cassette's recorded - response and assert the freshly captured request matches the cassette - (``wire`` plane). - -No production code changes: the executor's own connection ``endpoint`` is set to -``server.base_url`` and its OpenAI SDK POSTs there exactly as it would to the -real base URL. -""" - -from __future__ import annotations - -import http.server -import json -import threading -from typing import Any - - -class _RecordingHandler(http.server.BaseHTTPRequestHandler): - """Records each POST body and replies with the server's canned response.""" - - # Silence the default stderr access log so conformance output stays clean. - def log_message(self, *args: Any) -> None: # noqa: D401 - stdlib override - return - - def do_POST(self) -> None: # noqa: N802 - stdlib handler name - length = int(self.headers.get("Content-Length", "0") or "0") - raw = self.rfile.read(length) if length else b"" - try: - body: Any = json.loads(raw) if raw else {} - except json.JSONDecodeError: - body = {"__raw__": raw.decode("utf-8", "replace")} - server: Any = self.server - server.captured.append({"method": "POST", "path": self.path, "body": body}) - - payload = json.dumps(server.response_body).encode("utf-8") - self.send_response(server.response_status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - -class CassetteReplayServer: - """A background HTTP server that replays one response and records requests. - - Use as a context manager. ``base_url`` is the redirect target to hand the - provider SDK; ``last_request`` is the most recently captured outbound request - (the wire plane's observed value). - """ - - def __init__(self, response_body: Any, response_status: int = 200) -> None: - self._response_body = response_body - self._response_status = response_status - self._httpd: http.server.HTTPServer | None = None - self._thread: threading.Thread | None = None - self.captured: list[dict[str, Any]] = [] - - def __enter__(self) -> CassetteReplayServer: - httpd = http.server.HTTPServer(("127.0.0.1", 0), _RecordingHandler) - # Stash server-scoped state the handler reads off ``self.server``. - httpd.captured = self.captured # type: ignore[attr-defined] - httpd.response_body = self._response_body # type: ignore[attr-defined] - httpd.response_status = self._response_status # type: ignore[attr-defined] - self._httpd = httpd - self._thread = threading.Thread(target=httpd.serve_forever, daemon=True) - self._thread.start() - return self - - def __exit__(self, *exc: Any) -> None: - if self._httpd is not None: - self._httpd.shutdown() - self._httpd.server_close() - if self._thread is not None: - self._thread.join(timeout=2) - - @property - def base_url(self) -> str: - assert self._httpd is not None, "server not started" - host, port = self._httpd.server_address[:2] - return f"http://{host}:{port}" - - @property - def last_request(self) -> dict[str, Any] | None: - return self.captured[-1] if self.captured else None diff --git a/runtime/python/prompty/tests/model/cassettes/openai_chat_drill.json b/runtime/python/prompty/tests/model/cassettes/openai_chat_drill.json deleted file mode 100644 index 8afdc52e0..000000000 --- a/runtime/python/prompty/tests/model/cassettes/openai_chat_drill.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "_comment": "Service-drill cassette for LiveChatConformance.complete/openai_chat_drill. Consumer-owned roster: NOT modeled in TypeSpec/Typra. Regenerate with `uv run python tests/model/record_cassette.py`. `request.body` is the wire plane the drill adapter asserts; `response.body` is replayed as the seam bytes.", - "recordedAt": "2026-09-04T22:41:20Z", - "request": { - "body": { - "max_completion_tokens": 16, - "messages": [ - { - "content": "Say hello in exactly one word.", - "role": "user" - } - ], - "model": "gpt-4o-mini", - "temperature": 0 - }, - "method": "POST", - "path": "/chat/completions" - }, - "response": { - "body": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "logprobs": null, - "message": { - "content": "Hello", - "role": "assistant" - } - } - ], - "created": 1700000000, - "id": "chatcmpl-drill-openai-0001", - "model": "gpt-4o-mini", - "object": "chat.completion", - "usage": { - "completion_tokens": 1, - "prompt_tokens": 12, - "total_tokens": 13 - } - }, - "status": 200 - } -} diff --git a/runtime/python/prompty/tests/model/record_cassette.py b/runtime/python/prompty/tests/model/record_cassette.py deleted file mode 100644 index a2572573f..000000000 --- a/runtime/python/prompty/tests/model/record_cassette.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Cassette-capture tooling for the service-drill vector (NOT a pytest test). - -Run this to (re)generate ``cassettes/openai_chat_drill.json``. It synthesizes a -schema-valid OpenAI ``ChatCompletion`` response, serves it from a local -``CassetteReplayServer``, drives the REAL executor once through the same code -path the drill adapter uses (``_drill_execute``), and freezes the captured -outbound request as the cassette's ``request`` plane. The recorded response is -the ``response`` plane. - -This lives in CI/dev tooling, NEVER in the assertion path: it does not diff or -auto-rewrite during conformance. The ``request`` it records is exactly what the -drill's wire plane later asserts against, so capturing (rather than -hand-fabricating) keeps the wire contract honest and reproducible. - -Usage (from ``runtime/python/prompty``): - - uv run python tests/model/record_cassette.py - -The prefix is deliberately ``record_`` (not ``test_``) so pytest ignores it. -""" - -from __future__ import annotations - -import json -import os -import sys -import time -from pathlib import Path - -_HERE = Path(__file__).resolve().parent -if str(_HERE) not in sys.path: - sys.path.insert(0, str(_HERE)) - -from _drill_replay import CassetteReplayServer # noqa: E402 -from vector_adapters import _DRILL_REPLAY_API_KEY, _drill_execute # noqa: E402 - -# Mirror the drill vector's `input` (schema/model/conformance/vectors/drill.tsp). -# The apiKey/endpoint refs are intentionally absent here -- capture always binds -# the transport to the local replay server, never a real provider. -_DRILL_INPUT: dict = { - "provider": "openai", - "model": "gpt-4o-mini", - "messages": [{"role": "user", "content": "Say hello in exactly one word."}], - "options": {"temperature": 0, "maxOutputTokens": 16}, -} - - -def _synthesize_response() -> dict: - """A schema-valid OpenAI ChatCompletion the SDK can parse into a response.""" - return { - "id": "chatcmpl-drill-openai-0001", - "object": "chat.completion", - "created": 1_700_000_000, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "Hello"}, - "finish_reason": "stop", - "logprobs": None, - } - ], - "usage": {"prompt_tokens": 12, "completion_tokens": 1, "total_tokens": 13}, - } - - -def main() -> None: - response_body = _synthesize_response() - - with CassetteReplayServer(response_body) as server: - raw = _drill_execute(_DRILL_INPUT, endpoint=server.base_url, api_key=_DRILL_REPLAY_API_KEY) - if raw is None: - raise SystemExit("capture failed: executor returned no response") - captured = server.last_request - if captured is None: - raise SystemExit("capture failed: no request reached the replay server") - - cassette = { - "_comment": ( - "Service-drill cassette for LiveChatConformance.complete/openai_chat_drill. " - "Consumer-owned roster: NOT modeled in TypeSpec/Typra. Regenerate with " - "`uv run python tests/model/record_cassette.py`. `request.body` is the wire " - "plane the drill adapter asserts; `response.body` is replayed as the seam bytes." - ), - "recordedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - "request": { - "method": captured.get("method"), - "path": captured.get("path"), - "body": captured.get("body"), - }, - "response": { - "status": 200, - "body": response_body, - }, - } - - out_dir = _HERE / "cassettes" - out_dir.mkdir(exist_ok=True) - out_path = out_dir / "openai_chat_drill.json" - out_path.write_text(json.dumps(cassette, indent=2, sort_keys=True) + "\n", encoding="utf-8") - rel = os.path.relpath(out_path, Path.cwd()) - print(f"wrote {rel}") - print(f" request.body = {json.dumps(cassette['request']['body'], sort_keys=True)}") - - -if __name__ == "__main__": - main() diff --git a/runtime/python/prompty/tests/model/test_capability_guard_reference.py b/runtime/python/prompty/tests/model/test_capability_guard_reference.py index 97e89487d..fad9eebd4 100644 --- a/runtime/python/prompty/tests/model/test_capability_guard_reference.py +++ b/runtime/python/prompty/tests/model/test_capability_guard_reference.py @@ -6,8 +6,7 @@ * the ``VECTOR_CAPABILITIES`` predicates in ``vector_adapters`` (present/absent), * a faithful LOCAL REPLICA of the emitter's two-pass guard so the skip/hard-fail semantics are verified without waiting for a regenerated harness, and -* the structure-not-content ``_drill_normalize`` projection (the drill's - semantic-plane reducer). +* the structure-not-content ``_live_chat_normalize`` projection. The generated harness (``test_vector_conformance.py``) only invokes the guard once a schema vector declares ``requires`` AND the emitter pin is bumped to @@ -208,7 +207,7 @@ def _openai_response(role: str = "assistant", content: str = "Hello", finish: st class TestLiveChatNormalize: def test_openai_shape_reduces_to_structure(self) -> None: - result = va._drill_normalize(_openai_response(), _context()) + result = va._live_chat_normalize(_openai_response(), _context()) assert result == { "role": "assistant", "contentNonEmpty": True, @@ -216,11 +215,11 @@ def test_openai_shape_reduces_to_structure(self) -> None: } def test_empty_content_flags_false(self) -> None: - result = va._drill_normalize(_openai_response(content=" "), _context()) + result = va._live_chat_normalize(_openai_response(content=" "), _context()) assert result["contentNonEmpty"] is False def test_finish_reason_outside_enum_flags_false(self) -> None: - result = va._drill_normalize(_openai_response(finish="explode"), _context()) + result = va._live_chat_normalize(_openai_response(finish="explode"), _context()) assert result["finishReasonInEnum"] is False @pytest.mark.parametrize( @@ -228,7 +227,7 @@ def test_finish_reason_outside_enum_flags_false(self) -> None: ["stop", "length", "tool_calls", "content_filter", "function_call"], ) def test_all_canonical_finish_reasons_pass(self, finish: str) -> None: - result = va._drill_normalize(_openai_response(finish=finish), _context()) + result = va._live_chat_normalize(_openai_response(finish=finish), _context()) assert result["finishReasonInEnum"] is True def test_anthropic_shape_maps_stop_reason(self) -> None: @@ -238,7 +237,7 @@ def test_anthropic_shape_maps_stop_reason(self) -> None: response.role = "assistant" response.content = [block] response.stop_reason = "end_turn" - result = va._drill_normalize(response, _context()) + result = va._live_chat_normalize(response, _context()) assert result == { "role": "assistant", "contentNonEmpty": True, @@ -266,8 +265,8 @@ def test_openai_live_chat_structure(self) -> None: "options": {"temperature": 0, "maxOutputTokens": 16}, } context = _context(vector={"input": resolved}) - observed = va._drill_execute(resolved, endpoint=None, api_key=resolved["apiKey"]) - structure = va._drill_normalize(observed, context) + observed = va._live_chat_invoke(resolved, context) + structure = va._live_chat_normalize(observed, context) assert structure["role"] == "assistant" assert structure["contentNonEmpty"] is True assert structure["finishReasonInEnum"] is True diff --git a/runtime/python/prompty/tests/model/test_vector_conformance.py b/runtime/python/prompty/tests/model/test_vector_conformance.py index 6c5592067..0b8b8158b 100644 --- a/runtime/python/prompty/tests/model/test_vector_conformance.py +++ b/runtime/python/prompty/tests/model/test_vector_conformance.py @@ -537,60 +537,7 @@ async def test_vector_15_discoveryconformance_mapmodel_openai_model_finetune_no_ await run_vector("DiscoveryConformance", "mapModel", vector, False, _SEAM) -async def test_vector_16_livechatconformance_complete_openai_chat_drill(): - vector_json = r""" - { - "name": "openai_chat_drill", - "stage": "drill", - "provider": "openai", - "input": { - "provider": "openai", - "model": "gpt-4o-mini", - "apiKey": { - "$env": "OPENAI_API_KEY" - }, - "endpoint": { - "$env": "OPENAI_BASE_URL" - }, - "messages": [ - { - "role": "user", - "content": "Say hello in exactly one word." - } - ], - "options": { - "temperature": 0, - "maxOutputTokens": 16 - } - }, - "exchange": { - "transport": { - "baseUrl": { - "$env": "OPENAI_BASE_URL" - } - }, - "cassette": { - "$json": "cassettes/openai_chat_drill.json" - }, - "planes": [ - "transport", - "wire", - "semantic" - ] - }, - "expected": { - "role": "assistant", - "contentNonEmpty": true, - "finishReasonInEnum": true - }, - "operation": "complete" - } - """ - vector = json.loads(vector_json, strict=False) - await run_vector("LiveChatConformance", "complete", vector, False, _SEAM) - - -async def test_vector_17_loadconformance_load_basic_load(): +async def test_vector_16_loadconformance_load_basic_load(): vector_json = r""" { "name": "basic_load", @@ -659,7 +606,7 @@ async def test_vector_17_loadconformance_load_basic_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_18_loadconformance_load_connection_types_load(): +async def test_vector_17_loadconformance_load_connection_types_load(): vector_json = r""" { "name": "connection_types_load", @@ -693,7 +640,7 @@ async def test_vector_18_loadconformance_load_connection_types_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_19_loadconformance_load_embedding_load(): +async def test_vector_18_loadconformance_load_embedding_load(): vector_json = r""" { "name": "embedding_load", @@ -734,7 +681,7 @@ async def test_vector_19_loadconformance_load_embedding_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_20_loadconformance_load_empty_frontmatter_body_only(): +async def test_vector_19_loadconformance_load_empty_frontmatter_body_only(): vector_json = r""" { "name": "empty_frontmatter_body_only", @@ -759,7 +706,7 @@ async def test_vector_20_loadconformance_load_empty_frontmatter_body_only(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_21_loadconformance_load_env_default(): +async def test_vector_20_loadconformance_load_env_default(): vector_json = r""" { "name": "env_default", @@ -793,7 +740,7 @@ async def test_vector_21_loadconformance_load_env_default(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_22_loadconformance_load_env_missing_error(): +async def test_vector_21_loadconformance_load_env_missing_error(): vector_json = r""" { "name": "env_missing_error", @@ -822,7 +769,7 @@ async def test_vector_22_loadconformance_load_env_missing_error(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_23_loadconformance_load_env_resolution(): +async def test_vector_22_loadconformance_load_env_resolution(): vector_json = r""" { "name": "env_resolution", @@ -858,7 +805,7 @@ async def test_vector_23_loadconformance_load_env_resolution(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_24_loadconformance_load_file_reference_in_tree_allowed(): +async def test_vector_23_loadconformance_load_file_reference_in_tree_allowed(): vector_json = r""" { "name": "file_reference_in_tree_allowed", @@ -896,7 +843,7 @@ async def test_vector_24_loadconformance_load_file_reference_in_tree_allowed(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_25_loadconformance_load_file_reference_traversal_rejected(): +async def test_vector_24_loadconformance_load_file_reference_traversal_rejected(): vector_json = r""" { "name": "file_reference_traversal_rejected", @@ -929,7 +876,7 @@ async def test_vector_25_loadconformance_load_file_reference_traversal_rejected( await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_26_loadconformance_load_file_resolution(): +async def test_vector_25_loadconformance_load_file_resolution(): vector_json = r""" { "name": "file_resolution", @@ -969,7 +916,7 @@ async def test_vector_26_loadconformance_load_file_resolution(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_27_loadconformance_load_image_apitype_load(): +async def test_vector_26_loadconformance_load_image_apitype_load(): vector_json = r""" { "name": "image_apitype_load", @@ -1013,7 +960,7 @@ async def test_vector_27_loadconformance_load_image_apitype_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_28_loadconformance_load_input_scalar_shorthand(): +async def test_vector_27_loadconformance_load_input_scalar_shorthand(): vector_json = r""" { "name": "input_scalar_shorthand", @@ -1058,7 +1005,7 @@ async def test_vector_28_loadconformance_load_input_scalar_shorthand(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_29_loadconformance_load_input_validation_default_fill(): +async def test_vector_28_loadconformance_load_input_validation_default_fill(): vector_json = r""" { "name": "input_validation_default_fill", @@ -1090,7 +1037,7 @@ async def test_vector_29_loadconformance_load_input_validation_default_fill(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_30_loadconformance_load_input_validation_example_not_used(): +async def test_vector_29_loadconformance_load_input_validation_example_not_used(): vector_json = r""" { "name": "input_validation_example_not_used", @@ -1121,7 +1068,7 @@ async def test_vector_30_loadconformance_load_input_validation_example_not_used( await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_31_loadconformance_load_input_validation_optional_omit(): +async def test_vector_30_loadconformance_load_input_validation_optional_omit(): vector_json = r""" { "name": "input_validation_optional_omit", @@ -1151,7 +1098,7 @@ async def test_vector_31_loadconformance_load_input_validation_optional_omit(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_32_loadconformance_load_input_validation_required(): +async def test_vector_31_loadconformance_load_input_validation_required(): vector_json = r""" { "name": "input_validation_required", @@ -1182,7 +1129,7 @@ async def test_vector_32_loadconformance_load_input_validation_required(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_33_loadconformance_load_instructions_from_body(): +async def test_vector_32_loadconformance_load_instructions_from_body(): vector_json = r""" { "name": "instructions_from_body", @@ -1205,7 +1152,7 @@ async def test_vector_33_loadconformance_load_instructions_from_body(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_34_loadconformance_load_invalid_frontmatter_error(): +async def test_vector_33_loadconformance_load_invalid_frontmatter_error(): vector_json = r""" { "name": "invalid_frontmatter_error", @@ -1224,7 +1171,7 @@ async def test_vector_34_loadconformance_load_invalid_frontmatter_error(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_35_loadconformance_load_kind_always_prompt(): +async def test_vector_34_loadconformance_load_kind_always_prompt(): vector_json = r""" { "name": "kind_always_prompt", @@ -1243,7 +1190,7 @@ async def test_vector_35_loadconformance_load_kind_always_prompt(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_36_loadconformance_load_minimal_load(): +async def test_vector_35_loadconformance_load_minimal_load(): vector_json = r""" { "name": "minimal_load", @@ -1270,7 +1217,7 @@ async def test_vector_36_loadconformance_load_minimal_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_37_loadconformance_load_missing_file_error(): +async def test_vector_36_loadconformance_load_missing_file_error(): vector_json = r""" { "name": "missing_file_error", @@ -1289,7 +1236,7 @@ async def test_vector_37_loadconformance_load_missing_file_error(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_38_loadconformance_load_model_shorthand(): +async def test_vector_37_loadconformance_load_model_shorthand(): vector_json = r""" { "name": "model_shorthand", @@ -1315,7 +1262,7 @@ async def test_vector_38_loadconformance_load_model_shorthand(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_39_loadconformance_load_structured_outputs_load(): +async def test_vector_38_loadconformance_load_structured_outputs_load(): vector_json = r""" { "name": "structured_outputs_load", @@ -1381,7 +1328,7 @@ async def test_vector_39_loadconformance_load_structured_outputs_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_40_loadconformance_load_template_string_invalid(): +async def test_vector_39_loadconformance_load_template_string_invalid(): vector_json = r""" { "name": "template_string_invalid", @@ -1406,7 +1353,7 @@ async def test_vector_40_loadconformance_load_template_string_invalid(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_41_loadconformance_load_tools_custom_load(): +async def test_vector_40_loadconformance_load_tools_custom_load(): vector_json = r""" { "name": "tools_custom_load", @@ -1443,7 +1390,7 @@ async def test_vector_41_loadconformance_load_tools_custom_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_42_loadconformance_load_tools_function_load(): +async def test_vector_41_loadconformance_load_tools_function_load(): vector_json = r""" { "name": "tools_function_load", @@ -1509,7 +1456,7 @@ async def test_vector_42_loadconformance_load_tools_function_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_43_loadconformance_load_tools_mcp_load(): +async def test_vector_42_loadconformance_load_tools_mcp_load(): vector_json = r""" { "name": "tools_mcp_load", @@ -1548,7 +1495,7 @@ async def test_vector_43_loadconformance_load_tools_mcp_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_44_loadconformance_load_tools_openapi_load(): +async def test_vector_43_loadconformance_load_tools_openapi_load(): vector_json = r""" { "name": "tools_openapi_load", @@ -1587,7 +1534,7 @@ async def test_vector_44_loadconformance_load_tools_openapi_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_45_turnconformance_replay_max_iterations(): +async def test_vector_44_turnconformance_replay_max_iterations(): vector_json = r""" { "name": "max_iterations", @@ -1622,7 +1569,7 @@ async def test_vector_45_turnconformance_replay_max_iterations(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_46_turnconformance_replay_no_tool(): +async def test_vector_45_turnconformance_replay_no_tool(): vector_json = r""" { "name": "no_tool", @@ -1653,7 +1600,7 @@ async def test_vector_46_turnconformance_replay_no_tool(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_47_turnconformance_replay_permission_denied(): +async def test_vector_46_turnconformance_replay_permission_denied(): vector_json = r""" { "name": "permission_denied", @@ -1687,7 +1634,7 @@ async def test_vector_47_turnconformance_replay_permission_denied(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_48_turnconformance_replay_tool_failure(): +async def test_vector_47_turnconformance_replay_tool_failure(): vector_json = r""" { "name": "tool_failure", @@ -1723,7 +1670,7 @@ async def test_vector_48_turnconformance_replay_tool_failure(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_49_turnconformance_replay_tool_success(): +async def test_vector_48_turnconformance_replay_tool_success(): vector_json = r""" { "name": "tool_success", @@ -1759,7 +1706,7 @@ async def test_vector_49_turnconformance_replay_tool_success(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_50_turnconformance_run_assistant_tool_calls_metadata(): +async def test_vector_49_turnconformance_run_assistant_tool_calls_metadata(): vector_json = r""" { "name": "assistant_tool_calls_metadata", @@ -1911,7 +1858,7 @@ async def test_vector_50_turnconformance_run_assistant_tool_calls_metadata(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_51_turnconformance_run_async_tool_function(): +async def test_vector_50_turnconformance_run_async_tool_function(): vector_json = r""" { "name": "async_tool_function", @@ -2021,7 +1968,7 @@ async def test_vector_51_turnconformance_run_async_tool_function(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_52_turnconformance_run_bindings_injected(): +async def test_vector_51_turnconformance_run_bindings_injected(): vector_json = r""" { "name": "bindings_injected", @@ -2151,7 +2098,7 @@ async def test_vector_52_turnconformance_run_bindings_injected(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_53_turnconformance_run_cancellation_before_llm(): +async def test_vector_52_turnconformance_run_cancellation_before_llm(): vector_json = r""" { "name": "cancellation_before_llm", @@ -2212,7 +2159,7 @@ async def test_vector_53_turnconformance_run_cancellation_before_llm(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_54_turnconformance_run_cancellation_between_iterations(): +async def test_vector_53_turnconformance_run_cancellation_between_iterations(): vector_json = r""" { "name": "cancellation_between_iterations", @@ -2342,7 +2289,7 @@ async def test_vector_54_turnconformance_run_cancellation_between_iterations(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_55_turnconformance_run_cancellation_between_tools(): +async def test_vector_54_turnconformance_run_cancellation_between_tools(): vector_json = r""" { "name": "cancellation_between_tools", @@ -2481,7 +2428,7 @@ async def test_vector_55_turnconformance_run_cancellation_between_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_56_turnconformance_run_context_no_trim_when_fits(): +async def test_vector_55_turnconformance_run_context_no_trim_when_fits(): vector_json = r""" { "name": "context_no_trim_when_fits", @@ -2552,7 +2499,7 @@ async def test_vector_56_turnconformance_run_context_no_trim_when_fits(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_57_turnconformance_run_context_preserves_system_messages(): +async def test_vector_56_turnconformance_run_context_preserves_system_messages(): vector_json = r""" { "name": "context_preserves_system_messages", @@ -2659,7 +2606,7 @@ async def test_vector_57_turnconformance_run_context_preserves_system_messages() await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_58_turnconformance_run_context_trim_basic(): +async def test_vector_57_turnconformance_run_context_trim_basic(): vector_json = r""" { "name": "context_trim_basic", @@ -2817,7 +2764,7 @@ async def test_vector_58_turnconformance_run_context_trim_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_59_turnconformance_run_empty_tool_result(): +async def test_vector_58_turnconformance_run_empty_tool_result(): vector_json = r""" { "name": "empty_tool_result", @@ -2932,7 +2879,7 @@ async def test_vector_59_turnconformance_run_empty_tool_result(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_60_turnconformance_run_events_basic_tool_loop(): +async def test_vector_59_turnconformance_run_events_basic_tool_loop(): vector_json = r""" { "name": "events_basic_tool_loop", @@ -3078,7 +3025,7 @@ async def test_vector_60_turnconformance_run_events_basic_tool_loop(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_61_turnconformance_run_events_error_logged(): +async def test_vector_60_turnconformance_run_events_error_logged(): vector_json = r""" { "name": "events_error_logged", @@ -3204,7 +3151,7 @@ async def test_vector_61_turnconformance_run_events_error_logged(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_62_turnconformance_run_events_no_tools(): +async def test_vector_61_turnconformance_run_events_no_tools(): vector_json = r""" { "name": "events_no_tools", @@ -3282,7 +3229,7 @@ async def test_vector_62_turnconformance_run_events_no_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_63_turnconformance_run_guardrail_all_pass(): +async def test_vector_62_turnconformance_run_guardrail_all_pass(): vector_json = r""" { "name": "guardrail_all_pass", @@ -3405,7 +3352,7 @@ async def test_vector_63_turnconformance_run_guardrail_all_pass(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_64_turnconformance_run_guardrail_input_deny(): +async def test_vector_63_turnconformance_run_guardrail_input_deny(): vector_json = r""" { "name": "guardrail_input_deny", @@ -3461,7 +3408,7 @@ async def test_vector_64_turnconformance_run_guardrail_input_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_65_turnconformance_run_guardrail_output_deny(): +async def test_vector_64_turnconformance_run_guardrail_output_deny(): vector_json = r""" { "name": "guardrail_output_deny", @@ -3539,7 +3486,7 @@ async def test_vector_65_turnconformance_run_guardrail_output_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_66_turnconformance_run_guardrail_tool_deny(): +async def test_vector_65_turnconformance_run_guardrail_tool_deny(): vector_json = r""" { "name": "guardrail_tool_deny", @@ -3693,7 +3640,7 @@ async def test_vector_66_turnconformance_run_guardrail_tool_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_67_turnconformance_run_max_iterations_exceeded(): +async def test_vector_66_turnconformance_run_max_iterations_exceeded(): vector_json = r""" { "name": "max_iterations_exceeded", @@ -4203,7 +4150,7 @@ async def test_vector_67_turnconformance_run_max_iterations_exceeded(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_68_turnconformance_run_multi_turn_tool_calls(): +async def test_vector_67_turnconformance_run_multi_turn_tool_calls(): vector_json = r""" { "name": "multi_turn_tool_calls", @@ -4380,7 +4327,7 @@ async def test_vector_68_turnconformance_run_multi_turn_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_69_turnconformance_run_multiple_tool_calls_single_turn(): +async def test_vector_68_turnconformance_run_multiple_tool_calls_single_turn(): vector_json = r""" { "name": "multiple_tool_calls_single_turn", @@ -4561,7 +4508,7 @@ async def test_vector_69_turnconformance_run_multiple_tool_calls_single_turn(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_70_turnconformance_run_no_tool_calls(): +async def test_vector_69_turnconformance_run_no_tool_calls(): vector_json = r""" { "name": "no_tool_calls", @@ -4629,7 +4576,7 @@ async def test_vector_70_turnconformance_run_no_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_71_turnconformance_run_parallel_tools_basic(): +async def test_vector_70_turnconformance_run_parallel_tools_basic(): vector_json = r""" { "name": "parallel_tools_basic", @@ -4803,7 +4750,7 @@ async def test_vector_71_turnconformance_run_parallel_tools_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_72_turnconformance_run_parallel_tools_with_guardrail_deny(): +async def test_vector_71_turnconformance_run_parallel_tools_with_guardrail_deny(): vector_json = r""" { "name": "parallel_tools_with_guardrail_deny", @@ -4995,7 +4942,7 @@ async def test_vector_72_turnconformance_run_parallel_tools_with_guardrail_deny( await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_73_turnconformance_run_single_tool_call(): +async def test_vector_72_turnconformance_run_single_tool_call(): vector_json = r""" { "name": "single_tool_call", @@ -5142,7 +5089,7 @@ async def test_vector_73_turnconformance_run_single_tool_call(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_74_turnconformance_run_steering_inject_message(): +async def test_vector_73_turnconformance_run_steering_inject_message(): vector_json = r""" { "name": "steering_inject_message", @@ -5350,7 +5297,7 @@ async def test_vector_74_turnconformance_run_steering_inject_message(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_75_turnconformance_run_steering_multiple_messages(): +async def test_vector_74_turnconformance_run_steering_multiple_messages(): vector_json = r""" { "name": "steering_multiple_messages", @@ -5520,7 +5467,7 @@ async def test_vector_75_turnconformance_run_steering_multiple_messages(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_76_turnconformance_run_tool_not_registered_error(): +async def test_vector_75_turnconformance_run_tool_not_registered_error(): vector_json = r""" { "name": "tool_not_registered_error", @@ -5604,7 +5551,7 @@ async def test_vector_76_turnconformance_run_tool_not_registered_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_77_turnconformance_run_tool_result_message_format(): +async def test_vector_76_turnconformance_run_tool_result_message_format(): vector_json = r""" { "name": "tool_result_message_format", @@ -5725,7 +5672,7 @@ async def test_vector_77_turnconformance_run_tool_result_message_format(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_78_turnconformance_runturn_cancel_before_context(): +async def test_vector_77_turnconformance_runturn_cancel_before_context(): vector_json = r""" { "name": "cancel_before_context", @@ -5757,7 +5704,7 @@ async def test_vector_78_turnconformance_runturn_cancel_before_context(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_79_turnconformance_runturn_delegated_provider_state(): +async def test_vector_78_turnconformance_runturn_delegated_provider_state(): vector_json = r""" { "name": "delegated_provider_state", @@ -5824,7 +5771,7 @@ async def test_vector_79_turnconformance_runturn_delegated_provider_state(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_80_turnconformance_runturn_final_output(): +async def test_vector_79_turnconformance_runturn_final_output(): vector_json = r""" { "name": "final_output", @@ -5869,7 +5816,7 @@ async def test_vector_80_turnconformance_runturn_final_output(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_81_turnconformance_runturn_ordered_tool_round(): +async def test_vector_80_turnconformance_runturn_ordered_tool_round(): vector_json = r""" { "name": "ordered_tool_round", @@ -5960,7 +5907,7 @@ async def test_vector_81_turnconformance_runturn_ordered_tool_round(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_82_turnconformance_runturn_permission_denial_is_model_visible(): +async def test_vector_81_turnconformance_runturn_permission_denial_is_model_visible(): vector_json = r""" { "name": "permission_denial_is_model_visible", @@ -6011,7 +5958,7 @@ async def test_vector_82_turnconformance_runturn_permission_denial_is_model_visi await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_83_wireconformance_torequest_anthropic_image_format(): +async def test_vector_82_wireconformance_torequest_anthropic_image_format(): vector_json = r""" { "name": "anthropic_image_format", @@ -6081,7 +6028,7 @@ async def test_vector_83_wireconformance_torequest_anthropic_image_format(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_84_wireconformance_torequest_anthropic_max_tokens_required(): +async def test_vector_83_wireconformance_torequest_anthropic_max_tokens_required(): vector_json = r""" { "name": "anthropic_max_tokens_required", @@ -6136,7 +6083,7 @@ async def test_vector_84_wireconformance_torequest_anthropic_max_tokens_required await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_85_wireconformance_torequest_anthropic_options(): +async def test_vector_84_wireconformance_torequest_anthropic_options(): vector_json = r""" { "name": "anthropic_options", @@ -6205,7 +6152,7 @@ async def test_vector_85_wireconformance_torequest_anthropic_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_86_wireconformance_torequest_anthropic_system_separate(): +async def test_vector_85_wireconformance_torequest_anthropic_system_separate(): vector_json = r""" { "name": "anthropic_system_separate", @@ -6272,7 +6219,7 @@ async def test_vector_86_wireconformance_torequest_anthropic_system_separate(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_87_wireconformance_torequest_anthropic_tool_wire(): +async def test_vector_86_wireconformance_torequest_anthropic_tool_wire(): vector_json = r""" { "name": "anthropic_tool_wire", @@ -6357,7 +6304,7 @@ async def test_vector_87_wireconformance_torequest_anthropic_tool_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_88_wireconformance_torequest_chat_audio_mp3(): +async def test_vector_87_wireconformance_torequest_chat_audio_mp3(): vector_json = r""" { "name": "chat_audio_mp3", @@ -6415,7 +6362,7 @@ async def test_vector_88_wireconformance_torequest_chat_audio_mp3(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_89_wireconformance_torequest_chat_audio_part(): +async def test_vector_88_wireconformance_torequest_chat_audio_part(): vector_json = r""" { "name": "chat_audio_part", @@ -6473,7 +6420,7 @@ async def test_vector_89_wireconformance_torequest_chat_audio_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_90_wireconformance_torequest_chat_image_base64(): +async def test_vector_89_wireconformance_torequest_chat_image_base64(): vector_json = r""" { "name": "chat_image_base64", @@ -6537,7 +6484,7 @@ async def test_vector_90_wireconformance_torequest_chat_image_base64(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_91_wireconformance_torequest_chat_image_part(): +async def test_vector_90_wireconformance_torequest_chat_image_part(): vector_json = r""" { "name": "chat_image_part", @@ -6601,7 +6548,7 @@ async def test_vector_91_wireconformance_torequest_chat_image_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_92_wireconformance_torequest_chat_multipart_content(): +async def test_vector_91_wireconformance_torequest_chat_multipart_content(): vector_json = r""" { "name": "chat_multipart_content", @@ -6665,7 +6612,7 @@ async def test_vector_92_wireconformance_torequest_chat_multipart_content(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_93_wireconformance_torequest_chat_simple(): +async def test_vector_92_wireconformance_torequest_chat_simple(): vector_json = r""" { "name": "chat_simple", @@ -6727,7 +6674,7 @@ async def test_vector_93_wireconformance_torequest_chat_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_94_wireconformance_torequest_chat_single_text_optimized(): +async def test_vector_93_wireconformance_torequest_chat_single_text_optimized(): vector_json = r""" { "name": "chat_single_text_optimized", @@ -6776,7 +6723,7 @@ async def test_vector_94_wireconformance_torequest_chat_single_text_optimized(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_95_wireconformance_torequest_chat_with_options(): +async def test_vector_94_wireconformance_torequest_chat_with_options(): vector_json = r""" { "name": "chat_with_options", @@ -6844,7 +6791,7 @@ async def test_vector_95_wireconformance_torequest_chat_with_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_96_wireconformance_torequest_embedding_wire(): +async def test_vector_95_wireconformance_torequest_embedding_wire(): vector_json = r""" { "name": "embedding_wire", @@ -6888,7 +6835,7 @@ async def test_vector_96_wireconformance_torequest_embedding_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_97_wireconformance_torequest_image_wire(): +async def test_vector_96_wireconformance_torequest_image_wire(): vector_json = r""" { "name": "image_wire", @@ -6932,7 +6879,7 @@ async def test_vector_97_wireconformance_torequest_image_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_98_wireconformance_torequest_kind_to_json_type_mapping(): +async def test_vector_97_wireconformance_torequest_kind_to_json_type_mapping(): vector_json = r""" { "name": "kind_to_json_type_mapping", @@ -7051,7 +6998,7 @@ async def test_vector_98_wireconformance_torequest_kind_to_json_type_mapping(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_99_wireconformance_torequest_options_additional_properties(): +async def test_vector_98_wireconformance_torequest_options_additional_properties(): vector_json = r""" { "name": "options_additional_properties", @@ -7105,7 +7052,7 @@ async def test_vector_99_wireconformance_torequest_options_additional_properties await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_100_wireconformance_torequest_options_max_completion_tokens(): +async def test_vector_99_wireconformance_torequest_options_max_completion_tokens(): vector_json = r""" { "name": "options_max_completion_tokens", @@ -7157,7 +7104,7 @@ async def test_vector_100_wireconformance_torequest_options_max_completion_token await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_101_wireconformance_torequest_options_stop_sequences(): +async def test_vector_100_wireconformance_torequest_options_stop_sequences(): vector_json = r""" { "name": "options_stop_sequences", @@ -7215,7 +7162,7 @@ async def test_vector_101_wireconformance_torequest_options_stop_sequences(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_102_wireconformance_torequest_responses_simple(): +async def test_vector_101_wireconformance_torequest_responses_simple(): vector_json = r""" { "name": "responses_simple", @@ -7274,7 +7221,7 @@ async def test_vector_102_wireconformance_torequest_responses_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_103_wireconformance_torequest_responses_structured_output(): +async def test_vector_102_wireconformance_torequest_responses_structured_output(): vector_json = r""" { "name": "responses_structured_output", @@ -7361,7 +7308,7 @@ async def test_vector_103_wireconformance_torequest_responses_structured_output( await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_104_wireconformance_torequest_responses_with_tools(): +async def test_vector_103_wireconformance_torequest_responses_with_tools(): vector_json = r""" { "name": "responses_with_tools", @@ -7441,7 +7388,7 @@ async def test_vector_104_wireconformance_torequest_responses_with_tools(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_105_wireconformance_torequest_structured_output(): +async def test_vector_104_wireconformance_torequest_structured_output(): vector_json = r""" { "name": "structured_output", @@ -7524,7 +7471,7 @@ async def test_vector_105_wireconformance_torequest_structured_output(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_106_wireconformance_torequest_structured_output_nested_optional(): +async def test_vector_105_wireconformance_torequest_structured_output_nested_optional(): vector_json = r""" { "name": "structured_output_nested_optional", @@ -7626,7 +7573,7 @@ async def test_vector_106_wireconformance_torequest_structured_output_nested_opt await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_107_wireconformance_torequest_tools_bindings_stripped(): +async def test_vector_106_wireconformance_torequest_tools_bindings_stripped(): vector_json = r""" { "name": "tools_bindings_stripped", @@ -7718,7 +7665,7 @@ async def test_vector_107_wireconformance_torequest_tools_bindings_stripped(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_108_wireconformance_torequest_tools_function_wire(): +async def test_vector_107_wireconformance_torequest_tools_function_wire(): vector_json = r""" { "name": "tools_function_wire", @@ -7800,7 +7747,7 @@ async def test_vector_108_wireconformance_torequest_tools_function_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_109_wireconformance_torequest_tools_null_when_empty(): +async def test_vector_108_wireconformance_torequest_tools_null_when_empty(): vector_json = r""" { "name": "tools_null_when_empty", @@ -7849,7 +7796,7 @@ async def test_vector_109_wireconformance_torequest_tools_null_when_empty(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_110_wireconformance_torequest_tools_strict_mode(): +async def test_vector_109_wireconformance_torequest_tools_strict_mode(): vector_json = r""" { "name": "tools_strict_mode", diff --git a/runtime/python/prompty/tests/model/vector_adapters.py b/runtime/python/prompty/tests/model/vector_adapters.py index a96d84afb..8f9d3a448 100644 --- a/runtime/python/prompty/tests/model/vector_adapters.py +++ b/runtime/python/prompty/tests/model/vector_adapters.py @@ -983,27 +983,28 @@ def _live_provider_impls(provider: str) -> tuple[Any, Any]: raise ValueError(f"live-chat adapter: unknown provider {provider!r}") -def _drill_build_agent(resolved_input: dict, *, endpoint: str | None, api_key: str | None) -> Any: - """Construct a chat ``Agent`` bound to a specific transport ``endpoint``. - - The drill binds the transport seam via base-URL redirect, so ``endpoint`` and - ``api_key`` are supplied by the caller (a local cassette-replay server in - replay mode, the real base URL + key in live mode) rather than read from the - vector input. When ``endpoint`` is falsy the connection omits it so the SDK - falls back to the provider's default base URL (live mode without an override). - """ +def _build_live_agent(resolved_input: dict) -> Any: + """Construct a chat ``Agent`` from a resolved live-chat vector input.""" from prompty.model import Agent provider = (resolved_input.get("provider") or "openai").lower() - - connection: dict[str, Any] = {"kind": "key"} - if api_key: - connection["apiKey"] = api_key - if endpoint: - connection["endpoint"] = endpoint + api_key = resolved_input.get("apiKey") + endpoint = resolved_input.get("endpoint") + + if provider in ("foundry", "azure") and not api_key: + # Entra ID path: no key, DefaultAzureCredential drives auth. + connection: dict[str, Any] = {"kind": "foundry"} + if endpoint: + connection["endpoint"] = endpoint + else: + connection = {"kind": "key"} + if api_key: + connection["apiKey"] = api_key + if endpoint: + connection["endpoint"] = endpoint data: dict[str, Any] = { - "name": "drill-chat-vector", + "name": "live-chat-vector", "model": { "id": resolved_input.get("model") or "gpt-4o-mini", "provider": provider, @@ -1055,63 +1056,25 @@ def _extract_chat_structure(observed: Any) -> tuple[Any, Any, Any]: return role, content, finish -# --------------------------------------------------------------------------- -# LiveChatConformance.complete -- the service-drill adapter -# --------------------------------------------------------------------------- -# -# A "drill" is a conformance vector that crosses the REAL Executor transport -# seam. Unlike the pure conformance vectors that stop at the SDK edge, the drill -# binds the seam via base-URL redirect -- to a local ``CassetteReplayServer`` -# (replay mode) or the real base URL (live mode) -- and asserts three planes: -# -# 1. transport -- the seam was reached and returned bytes (executor got a -# response, and the replay server captured an inbound request). -# 2. wire -- the outbound provider request body matches the cassette's -# recorded request body (canonical equality; throw on mismatch). -# 3. semantic -- normalized executor output == the vector `expected`. This -# plane is asserted by the RUNNER via the registered ``normalize``, so the -# drill's semantic plane IS a conformance vector over the executor. -# -# In live mode the adapter additionally asserts live-semantic == replay-semantic -# (parity). ``requires`` is deliberately OMITTED from the drill vector -- the -# ``requires`` guard is all-or-nothing and would skip the WHOLE vector when creds -# are absent, defeating the replay plane. Instead the adapter self-gates the live -# plane on credential presence and always runs replay. -# -# Cassette bytes and the SDK transport binding are consumer-owned "roster" -- by -# design NOT modeled in TypeSpec/Typra. - -_DRILL_REPLAY_API_KEY = "sk-drill-replay-placeholder" - - -def _drill_canonical(value: Any) -> str: - """Order-insensitive canonical JSON for wire/parity equality checks.""" - return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) - +def _live_chat_invoke(resolved_input: dict, context: dict) -> Any: + """Call a real provider and return its raw chat response. -def _drill_build_messages(resolved_input: dict) -> list: + Only reached when the capability guard has already confirmed the required + credentials/capabilities are present, so this makes a genuine network call. + """ from prompty.core.types import Message, TextPart - return [ + messages_spec = resolved_input.get("messages") or [] + messages = [ Message(role=spec.get("role", "user"), parts=[TextPart(value=spec.get("content", ""))]) - for spec in (resolved_input.get("messages") or []) + for spec in messages_spec ] - - -def _drill_execute(resolved_input: dict, *, endpoint: str | None, api_key: str | None) -> Any: - """Drive the real executor with its transport bound to ``endpoint``. - - Returns the raw provider response. Shared by the replay path (``endpoint`` = - a ``CassetteReplayServer`` base URL) and the live path (``endpoint`` = the - real base URL or ``None`` for the SDK default). - """ - agent = _drill_build_agent(resolved_input, endpoint=endpoint, api_key=api_key) + agent = _build_live_agent(resolved_input) executor, _processor = _live_provider_impls(resolved_input.get("provider") or "openai") - messages = _drill_build_messages(resolved_input) return executor.execute(agent, messages) -def _drill_normalize_response(observed: Any) -> dict: +def _live_chat_normalize(observed: Any, context: dict) -> dict: """Project a raw chat response onto the canonical structural shape.""" role, content, finish = _extract_chat_structure(observed) return { @@ -1121,82 +1084,6 @@ def _drill_normalize_response(observed: Any) -> dict: } -def _drill_live_enabled(resolved_input: dict) -> bool: - """Live plane is enabled iff a base URL AND an API key resolved from env. - - ``$env`` refs resolve to ``""`` when unset, so this is false in replay-only - (no-creds) runs and true only when the vector's endpoint+apiKey env vars are - both populated. - """ - return bool(resolved_input.get("endpoint") and resolved_input.get("apiKey")) - - -def _drill_invoke(resolved_input: dict, context: dict) -> Any: - """Run the drill: bind transport, replay the cassette, assert 3 planes. - - Returns the raw replay response; the runner then applies ``normalize`` and - compares against the vector ``expected`` (the semantic plane). - """ - from _drill_replay import CassetteReplayServer - - resolve = context.get("resolveInput") - vector = context.get("vector") or {} - exchange_raw = vector.get("exchange") or {} - # The runner resolves refs only on `input`; the adapter owns `exchange` - # resolution (nested `$env` transport ref + `$json` cassette ref). - exchange = resolve(exchange_raw) if callable(resolve) else exchange_raw - - cassette = exchange.get("cassette") or {} - recorded_request = (cassette.get("request") or {}).get("body") - recorded_response = (cassette.get("response") or {}).get("body") - recorded_status = int((cassette.get("response") or {}).get("status", 200)) - if recorded_response is None: - raise AssertionError("drill cassette missing response.body") - - # --- Replay: bind the transport to the local cassette server ------------- - with CassetteReplayServer(recorded_response, recorded_status) as server: - raw = _drill_execute(resolved_input, endpoint=server.base_url, api_key=_DRILL_REPLAY_API_KEY) - - # Plane 1: transport -- the seam was reached and a request was captured. - captured = server.last_request - if captured is None: - raise AssertionError("drill transport plane: no request reached the replay server") - if raw is None: - raise AssertionError("drill transport plane: executor returned no response") - - # Plane 2: wire -- outbound request body matches the cassette request. - if recorded_request is not None: - observed_body = captured.get("body") - if _drill_canonical(observed_body) != _drill_canonical(recorded_request): - raise AssertionError( - "drill wire plane mismatch:\n" - f" observed={_drill_canonical(observed_body)}\n" - f" cassette={_drill_canonical(recorded_request)}" - ) - - # --- Live parity (self-gated): live-semantic must equal replay-semantic --- - if _drill_live_enabled(resolved_input): - live_raw = _drill_execute( - resolved_input, - endpoint=resolved_input.get("endpoint") or None, - api_key=resolved_input.get("apiKey"), - ) - if _drill_canonical(_drill_normalize_response(live_raw)) != _drill_canonical(_drill_normalize_response(raw)): - raise AssertionError( - "drill live/replay parity mismatch:\n" - f" live={_drill_normalize_response(live_raw)}\n" - f" replay={_drill_normalize_response(raw)}" - ) - - # Plane 3 (semantic) is asserted by the runner: normalize(raw) == expected. - return raw - - -def _drill_normalize(observed: Any, context: dict) -> dict: - """Registered semantic projection: raw response -> canonical structure.""" - return _drill_normalize_response(observed) - - # --------------------------------------------------------------------------- # Adapter registry # --------------------------------------------------------------------------- @@ -1209,7 +1096,7 @@ def _drill_normalize(observed: Any, context: dict) -> dict: "DiscoveryConformance.enrich": {"invoke": _discovery_enrich_invoke}, "DiscoveryConformance.mapModel": {"invoke": _discovery_map_invoke}, "TurnConformance.replay": {"invoke": _replay_invoke}, - "LiveChatConformance.complete": {"invoke": _drill_invoke, "normalize": _drill_normalize}, + "LiveChatConformance.complete": {"invoke": _live_chat_invoke, "normalize": _live_chat_normalize}, } # Contracts introduced/tightened by Typra 0.12.0 that the Python runtime does not diff --git a/runtime/rust/prompty/src/model/live_chat_conformance.rs b/runtime/rust/prompty/src/model/live_chat_conformance.rs deleted file mode 100644 index 15ccb14b0..000000000 --- a/runtime/rust/prompty/src/model/live_chat_conformance.rs +++ /dev/null @@ -1,21 +0,0 @@ -// -// Code generated by Typra emitter; DO NOT EDIT. - -#![allow( - unused_imports, - dead_code, - non_camel_case_types, - unused_variables, - unexpected_cfgs, - clippy::all -)] - -/// Conformance-only seam hosting the "service drill" vectors: a conformance vector that crosses the REAL Executor->Processor transport seam instead of stopping at the SDK edge. A drill runs against a live provider when credentials are present and REPLAYS a recorded cassette when they are not; the live<->replay parity is the gate. Like every other conformance seam this interface exists purely to carry -#[async_trait::async_trait] -pub trait LiveChatConformance: Send + Sync { - /// Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes - async fn complete( - &self, - input: &serde_json::Value, - ) -> Result>; -} diff --git a/runtime/rust/prompty/src/model/mod.rs b/runtime/rust/prompty/src/model/mod.rs index 5c8027190..84a3bea3b 100644 --- a/runtime/rust/prompty/src/model/mod.rs +++ b/runtime/rust/prompty/src/model/mod.rs @@ -46,9 +46,6 @@ pub use executor::*; pub mod host_tool_executor; pub use host_tool_executor::*; -pub mod live_chat_conformance; -pub use live_chat_conformance::*; - pub mod load_conformance; pub use load_conformance::*; diff --git a/runtime/rust/prompty/tests/model/vector_adapters.rs b/runtime/rust/prompty/tests/model/vector_adapters.rs index edab7f1c4..2cf961295 100644 --- a/runtime/rust/prompty/tests/model/vector_adapters.rs +++ b/runtime/rust/prompty/tests/model/vector_adapters.rs @@ -201,10 +201,7 @@ pub fn adapters() -> HashMap<&'static str, Adapter> { } pub fn waivers() -> HashMap<&'static str, &'static str> { - HashMap::from([( - "LiveChatConformance.complete", - "drill: consumer-owned transport-seam probe (Python/TS only)", - )]) + HashMap::new() } pub fn doubles() -> Value { diff --git a/runtime/rust/prompty/tests/model/vector_conformance_test.rs b/runtime/rust/prompty/tests/model/vector_conformance_test.rs index b1970acf1..7578fa9b4 100644 --- a/runtime/rust/prompty/tests/model/vector_conformance_test.rs +++ b/runtime/rust/prompty/tests/model/vector_conformance_test.rs @@ -173,665 +173,658 @@ async fn test_vector_15_discoveryconformance_mapmodel_openai_model_finetune_no_o } #[tokio::test] -async fn test_vector_16_livechatconformance_complete_openai_chat_drill() { - let vector: Value = serde_json::from_str("{\"name\":\"openai_chat_drill\",\"stage\":\"drill\",\"provider\":\"openai\",\"input\":{\"provider\":\"openai\",\"model\":\"gpt-4o-mini\",\"apiKey\":{\"$env\":\"OPENAI_API_KEY\"},\"endpoint\":{\"$env\":\"OPENAI_BASE_URL\"},\"messages\":[{\"role\":\"user\",\"content\":\"Say hello in exactly one word.\"}],\"options\":{\"temperature\":0,\"maxOutputTokens\":16}},\"exchange\":{\"transport\":{\"baseUrl\":{\"$env\":\"OPENAI_BASE_URL\"}},\"cassette\":{\"$json\":\"cassettes/openai_chat_drill.json\"},\"planes\":[\"transport\",\"wire\",\"semantic\"]},\"expected\":{\"role\":\"assistant\",\"contentNonEmpty\":true,\"finishReasonInEnum\":true},\"operation\":\"complete\"}") - .expect("failed to decode vector"); - vector_runner::vc_run_vector("LiveChatConformance", "complete", vector, false, vc_seam()).await; -} - -#[tokio::test] -async fn test_vector_17_loadconformance_load_basic_load() { +async fn test_vector_16_loadconformance_load_basic_load() { let vector: Value = serde_json::from_str("{\"name\":\"basic_load\",\"description\":\"Load basic.prompty with env var resolution and verify all PromptAgent fields\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"basic-prompt\",\"description\":\"A basic prompt for testing\",\"metadata\":{\"authors\":[\"testauthor\"]},\"model\":{\"id\":\"gpt-4\",\"provider\":\"openai\",\"apiType\":\"chat\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"},\"options\":{\"temperature\":0.7,\"maxOutputTokens\":1000}},\"inputs\":[{\"name\":\"firstName\",\"kind\":\"string\",\"default\":\"Jane\"},{\"name\":\"lastName\",\"kind\":\"string\",\"default\":\"Doe\"},{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What is the meaning of life?\"}],\"template\":{\"format\":{\"kind\":\"jinja2\"},\"parser\":{\"kind\":\"prompty\"}},\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_18_loadconformance_load_connection_types_load() { +async fn test_vector_17_loadconformance_load_connection_types_load() { let vector: Value = serde_json::from_str("{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_19_loadconformance_load_embedding_load() { +async fn test_vector_18_loadconformance_load_embedding_load() { let vector: Value = serde_json::from_str("{\"name\":\"embedding_load\",\"description\":\"Load embedding.prompty and verify apiType=embedding with correct model id\",\"stage\":\"load\",\"input\":{\"fixture\":\"embedding.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"embedding\",\"model\":{\"id\":\"text-embedding-3-small\",\"provider\":\"openai\",\"apiType\":\"embedding\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"}},\"inputs\":[{\"name\":\"text\",\"kind\":\"string\",\"default\":\"Hello world\"}],\"instructions\":\"{{text}}\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_20_loadconformance_load_empty_frontmatter_body_only() { +async fn test_vector_19_loadconformance_load_empty_frontmatter_body_only() { let vector: Value = serde_json::from_str("{\"name\":\"empty_frontmatter_body_only\",\"description\":\"Empty frontmatter with body \u{2014} instructions extracted, defaults applied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"empty-fm\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"empty-fm\",\"model\":null,\"inputs\":null,\"tools\":null},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_21_loadconformance_load_env_default() { +async fn test_vector_20_loadconformance_load_env_default() { let vector: Value = serde_json::from_str("{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_22_loadconformance_load_env_missing_error() { +async fn test_vector_21_loadconformance_load_env_missing_error() { let vector: Value = serde_json::from_str("{\"name\":\"env_missing_error\",\"description\":\"${env:NONEXISTENT} with no default and var not set raises an error\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-error-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:NONEXISTENT}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"env_var_not_set\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_23_loadconformance_load_env_resolution() { +async fn test_vector_22_loadconformance_load_env_resolution() { let vector: Value = serde_json::from_str("{\"name\":\"env_resolution\",\"description\":\"${env:MY_VAR} resolves to the environment variable value when set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MY_VAR}\"}}},\"env\":{\"MY_VAR\":\"hello\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"hello\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_24_loadconformance_load_file_reference_in_tree_allowed() { +async fn test_vector_23_loadconformance_load_file_reference_in_tree_allowed() { let vector: Value = serde_json::from_str("{\"name\":\"file_reference_in_tree_allowed\",\"description\":\"A ${file:sub/data.json} reference that stays within the prompt file's directory tree resolves to the parsed content (containment control alongside the traversal-reject vector)\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-in-tree-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:sub/data.json}\"}},\"files\":{\"sub/data.json\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\",\"apiKey\":\"ok\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_25_loadconformance_load_file_reference_traversal_rejected() { +async fn test_vector_24_loadconformance_load_file_reference_traversal_rejected() { let vector: Value = serde_json::from_str("{\"name\":\"file_reference_traversal_rejected\",\"description\":\"A ${file:../secret.json} reference that escapes the prompt file's directory via `..` is rejected even though the target file exists, preventing arbitrary file disclosure (GHSA-7pfc-5v9r-j299). Uses forward-slash separators so the vector holds on every OS.\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-traversal-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:../secret.json}\"}},\"files\":{\"../secret.json\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"leaked\"}}},\"expectedError\":{\"kind\":\"file_reference\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_26_loadconformance_load_file_resolution() { +async fn test_vector_25_loadconformance_load_file_resolution() { let vector: Value = serde_json::from_str("{\"name\":\"file_resolution\",\"description\":\"${file:shared_connection.json} resolves to the parsed JSON content of the referenced file\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"file-ref-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:shared_connection.json}\"}},\"env\":{\"OPENAI_API_KEY\":\"shared-key\"},\"files\":{\"shared_connection.json\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\",\"apiKey\":\"shared-key\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_27_loadconformance_load_image_apitype_load() { +async fn test_vector_26_loadconformance_load_image_apitype_load() { let vector: Value = serde_json::from_str("{\"name\":\"image_apitype_load\",\"description\":\"An image-generation prompt loads with apiType image and passthrough additionalProperties model options\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}}},\"expected\":{\"kind\":\"prompt\",\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_28_loadconformance_load_input_scalar_shorthand() { +async fn test_vector_27_loadconformance_load_input_scalar_shorthand() { let vector: Value = serde_json::from_str("{\"name\":\"input_scalar_shorthand\",\"description\":\"Scalar input values create typed Property with default set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"scalar-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]}},\"expected\":{\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_29_loadconformance_load_input_validation_default_fill() { +async fn test_vector_28_loadconformance_load_input_validation_default_fill() { let vector: Value = serde_json::from_str("{\"name\":\"input_validation_default_fill\",\"description\":\"An input with a default value is filled in when no value is provided\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"default-fill-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"name\",\"kind\":\"string\",\"default\":\"world\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{\"name\":\"world\"}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_30_loadconformance_load_input_validation_example_not_used() { +async fn test_vector_29_loadconformance_load_input_validation_example_not_used() { let vector: Value = serde_json::from_str("{\"name\":\"input_validation_example_not_used\",\"description\":\"An input with example='sample' but no default must NOT use the example at runtime\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"example-not-used-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"required\":false,\"example\":\"sample\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_31_loadconformance_load_input_validation_optional_omit() { +async fn test_vector_30_loadconformance_load_input_validation_optional_omit() { let vector: Value = serde_json::from_str("{\"name\":\"input_validation_optional_omit\",\"description\":\"A non-required input with no default and no value is omitted (not an error)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"optional-omit-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"nickname\",\"kind\":\"string\",\"required\":false}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_32_loadconformance_load_input_validation_required() { +async fn test_vector_31_loadconformance_load_input_validation_required() { let vector: Value = serde_json::from_str("{\"name\":\"input_validation_required\",\"description\":\"A required input with no default and no value provided raises ValueError\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"required-input-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},\"inputs\":{}},\"expectedError\":{\"kind\":\"missing_required_input\",\"field\":\"city\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_33_loadconformance_load_instructions_from_body() { +async fn test_vector_32_loadconformance_load_instructions_from_body() { let vector: Value = serde_json::from_str("{\"name\":\"instructions_from_body\",\"description\":\"The markdown body after the closing --- becomes the instructions field\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_34_loadconformance_load_invalid_frontmatter_error() { +async fn test_vector_33_loadconformance_load_invalid_frontmatter_error() { let vector: Value = serde_json::from_str("{\"name\":\"invalid_frontmatter_error\",\"description\":\"Malformed YAML in frontmatter raises a parse error\",\"stage\":\"load\",\"input\":{\"frontmatter_raw\":\"---\\nname: [invalid\\n---\\nHello\"},\"expectedError\":{\"kind\":\"invalid_frontmatter\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_35_loadconformance_load_kind_always_prompt() { +async fn test_vector_34_loadconformance_load_kind_always_prompt() { let vector: Value = serde_json::from_str("{\"name\":\"kind_always_prompt\",\"description\":\"The loader always injects kind='prompt' \u{2014} .prompty files never specify kind themselves\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_36_loadconformance_load_minimal_load() { +async fn test_vector_35_loadconformance_load_minimal_load() { let vector: Value = serde_json::from_str("{\"name\":\"minimal_load\",\"description\":\"Load minimal.prompty with model shorthand and bare-minimum frontmatter\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"minimal\",\"model\":{\"id\":\"gpt-4\"},\"instructions\":\"system:\\nHello world.\",\"inputs\":null,\"outputs\":null,\"tools\":null},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_37_loadconformance_load_missing_file_error() { +async fn test_vector_36_loadconformance_load_missing_file_error() { let vector: Value = serde_json::from_str("{\"name\":\"missing_file_error\",\"description\":\"Loading a nonexistent .prompty file raises FileNotFoundError\",\"stage\":\"load\",\"input\":{\"fixture\":\"nonexistent.prompty\"},\"expectedError\":{\"kind\":\"file_not_found\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_38_loadconformance_load_model_shorthand() { +async fn test_vector_37_loadconformance_load_model_shorthand() { let vector: Value = serde_json::from_str("{\"name\":\"model_shorthand\",\"description\":\"String shorthand 'model: gpt-4o' expands to a Model object with id set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"test\",\"model\":\"gpt-4o\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"test\",\"model\":{\"id\":\"gpt-4o\"}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_39_loadconformance_load_structured_outputs_load() { +async fn test_vector_38_loadconformance_load_structured_outputs_load() { let vector: Value = serde_json::from_str("{\"name\":\"structured_outputs_load\",\"description\":\"Outputs load as a list of properties preserving kind, description, and required flags\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"structured-output\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]}},\"expected\":{\"kind\":\"prompt\",\"name\":\"structured-output\",\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_40_loadconformance_load_template_string_invalid() { +async fn test_vector_39_loadconformance_load_template_string_invalid() { let vector: Value = serde_json::from_str("{\"name\":\"template_string_invalid\",\"description\":\"Template as a bare string is not valid v2 \u{2014} must be an object with format/parser\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"shorthand-test\",\"model\":{\"id\":\"gpt-4\"},\"template\":\"jinja2\"}},\"expectedError\":{\"kind\":\"invalid_template\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_41_loadconformance_load_tools_custom_load() { +async fn test_vector_40_loadconformance_load_tools_custom_load() { let vector: Value = serde_json::from_str("{\"name\":\"tools_custom_load\",\"description\":\"Load a prompty with an unknown tool kind \u{2014} falls through to CustomTool\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"custom-tool-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://custom.example.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\"}]},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_42_loadconformance_load_tools_function_load() { +async fn test_vector_41_loadconformance_load_tools_function_load() { let vector: Value = serde_json::from_str("{\"name\":\"tools_function_load\",\"description\":\"Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings\",\"stage\":\"load\",\"input\":{\"fixture\":\"tools_function.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"function-tools\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather in a given location\",\"strict\":true,\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"description\":\"The city and state, e.g. San Francisco, CA\"},{\"name\":\"unit\",\"kind\":\"string\",\"description\":\"Temperature unit\",\"enumValues\":[\"celsius\",\"fahrenheit\"]}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"inputs\":[{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What's the weather in Paris?\"},{\"name\":\"preferred_unit\",\"kind\":\"string\",\"default\":\"celsius\"}],\"instructions\":\"system:\\nYou are a helpful assistant with access to tools.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_43_loadconformance_load_tools_mcp_load() { +async fn test_vector_42_loadconformance_load_tools_mcp_load() { let vector: Value = serde_json::from_str("{\"name\":\"tools_mcp_load\",\"description\":\"Load a prompty with an MCP tool \u{2014} kind, serverName, connection preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"mcp-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-mcp\"}}]}},\"expected\":{\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\"}]},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_44_loadconformance_load_tools_openapi_load() { +async fn test_vector_43_loadconformance_load_tools_openapi_load() { let vector: Value = serde_json::from_str("{\"name\":\"tools_openapi_load\",\"description\":\"Load a prompty with an OpenAPI tool \u{2014} specification path preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"openapi-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://api.weather.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\"}]},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_45_turnconformance_replay_max_iterations() { +async fn test_vector_44_turnconformance_replay_max_iterations() { let vector: Value = serde_json::from_str("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_46_turnconformance_replay_no_tool() { +async fn test_vector_45_turnconformance_replay_no_tool() { let vector: Value = serde_json::from_str("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_47_turnconformance_replay_permission_denied() { +async fn test_vector_46_turnconformance_replay_permission_denied() { let vector: Value = serde_json::from_str("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_48_turnconformance_replay_tool_failure() { +async fn test_vector_47_turnconformance_replay_tool_failure() { let vector: Value = serde_json::from_str("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_49_turnconformance_replay_tool_success() { +async fn test_vector_48_turnconformance_replay_tool_success() { let vector: Value = serde_json::from_str("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_50_turnconformance_run_assistant_tool_calls_metadata() { +async fn test_vector_49_turnconformance_run_assistant_tool_calls_metadata() { let vector: Value = serde_json::from_str("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_51_turnconformance_run_async_tool_function() { +async fn test_vector_50_turnconformance_run_async_tool_function() { let vector: Value = serde_json::from_str("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_52_turnconformance_run_bindings_injected() { +async fn test_vector_51_turnconformance_run_bindings_injected() { let vector: Value = serde_json::from_str("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_53_turnconformance_run_cancellation_before_llm() { +async fn test_vector_52_turnconformance_run_cancellation_before_llm() { let vector: Value = serde_json::from_str("{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_54_turnconformance_run_cancellation_between_iterations() { +async fn test_vector_53_turnconformance_run_cancellation_between_iterations() { let vector: Value = serde_json::from_str("{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_55_turnconformance_run_cancellation_between_tools() { +async fn test_vector_54_turnconformance_run_cancellation_between_tools() { let vector: Value = serde_json::from_str("{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_56_turnconformance_run_context_no_trim_when_fits() { +async fn test_vector_55_turnconformance_run_context_no_trim_when_fits() { let vector: Value = serde_json::from_str("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_57_turnconformance_run_context_preserves_system_messages() { +async fn test_vector_56_turnconformance_run_context_preserves_system_messages() { let vector: Value = serde_json::from_str("{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_58_turnconformance_run_context_trim_basic() { +async fn test_vector_57_turnconformance_run_context_trim_basic() { let vector: Value = serde_json::from_str("{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_59_turnconformance_run_empty_tool_result() { +async fn test_vector_58_turnconformance_run_empty_tool_result() { let vector: Value = serde_json::from_str("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_60_turnconformance_run_events_basic_tool_loop() { +async fn test_vector_59_turnconformance_run_events_basic_tool_loop() { let vector: Value = serde_json::from_str("{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_61_turnconformance_run_events_error_logged() { +async fn test_vector_60_turnconformance_run_events_error_logged() { let vector: Value = serde_json::from_str("{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_62_turnconformance_run_events_no_tools() { +async fn test_vector_61_turnconformance_run_events_no_tools() { let vector: Value = serde_json::from_str("{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_63_turnconformance_run_guardrail_all_pass() { +async fn test_vector_62_turnconformance_run_guardrail_all_pass() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_64_turnconformance_run_guardrail_input_deny() { +async fn test_vector_63_turnconformance_run_guardrail_input_deny() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_65_turnconformance_run_guardrail_output_deny() { +async fn test_vector_64_turnconformance_run_guardrail_output_deny() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_66_turnconformance_run_guardrail_tool_deny() { +async fn test_vector_65_turnconformance_run_guardrail_tool_deny() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_67_turnconformance_run_max_iterations_exceeded() { +async fn test_vector_66_turnconformance_run_max_iterations_exceeded() { let vector: Value = serde_json::from_str("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_68_turnconformance_run_multi_turn_tool_calls() { +async fn test_vector_67_turnconformance_run_multi_turn_tool_calls() { let vector: Value = serde_json::from_str("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_69_turnconformance_run_multiple_tool_calls_single_turn() { +async fn test_vector_68_turnconformance_run_multiple_tool_calls_single_turn() { let vector: Value = serde_json::from_str("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_70_turnconformance_run_no_tool_calls() { +async fn test_vector_69_turnconformance_run_no_tool_calls() { let vector: Value = serde_json::from_str("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_71_turnconformance_run_parallel_tools_basic() { +async fn test_vector_70_turnconformance_run_parallel_tools_basic() { let vector: Value = serde_json::from_str("{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_72_turnconformance_run_parallel_tools_with_guardrail_deny() { +async fn test_vector_71_turnconformance_run_parallel_tools_with_guardrail_deny() { let vector: Value = serde_json::from_str("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_73_turnconformance_run_single_tool_call() { +async fn test_vector_72_turnconformance_run_single_tool_call() { let vector: Value = serde_json::from_str("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_74_turnconformance_run_steering_inject_message() { +async fn test_vector_73_turnconformance_run_steering_inject_message() { let vector: Value = serde_json::from_str("{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_75_turnconformance_run_steering_multiple_messages() { +async fn test_vector_74_turnconformance_run_steering_multiple_messages() { let vector: Value = serde_json::from_str("{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_76_turnconformance_run_tool_not_registered_error() { +async fn test_vector_75_turnconformance_run_tool_not_registered_error() { let vector: Value = serde_json::from_str("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_77_turnconformance_run_tool_result_message_format() { +async fn test_vector_76_turnconformance_run_tool_result_message_format() { let vector: Value = serde_json::from_str("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_78_turnconformance_runturn_cancel_before_context() { +async fn test_vector_77_turnconformance_runturn_cancel_before_context() { let vector: Value = serde_json::from_str("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_79_turnconformance_runturn_delegated_provider_state() { +async fn test_vector_78_turnconformance_runturn_delegated_provider_state() { let vector: Value = serde_json::from_str("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_80_turnconformance_runturn_final_output() { +async fn test_vector_79_turnconformance_runturn_final_output() { let vector: Value = serde_json::from_str("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_81_turnconformance_runturn_ordered_tool_round() { +async fn test_vector_80_turnconformance_runturn_ordered_tool_round() { let vector: Value = serde_json::from_str("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_82_turnconformance_runturn_permission_denial_is_model_visible() { +async fn test_vector_81_turnconformance_runturn_permission_denial_is_model_visible() { let vector: Value = serde_json::from_str("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_83_wireconformance_torequest_anthropic_image_format() { +async fn test_vector_82_wireconformance_torequest_anthropic_image_format() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_84_wireconformance_torequest_anthropic_max_tokens_required() { +async fn test_vector_83_wireconformance_torequest_anthropic_max_tokens_required() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_85_wireconformance_torequest_anthropic_options() { +async fn test_vector_84_wireconformance_torequest_anthropic_options() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_86_wireconformance_torequest_anthropic_system_separate() { +async fn test_vector_85_wireconformance_torequest_anthropic_system_separate() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_87_wireconformance_torequest_anthropic_tool_wire() { +async fn test_vector_86_wireconformance_torequest_anthropic_tool_wire() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_88_wireconformance_torequest_chat_audio_mp3() { +async fn test_vector_87_wireconformance_torequest_chat_audio_mp3() { let vector: Value = serde_json::from_str("{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_89_wireconformance_torequest_chat_audio_part() { +async fn test_vector_88_wireconformance_torequest_chat_audio_part() { let vector: Value = serde_json::from_str("{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_90_wireconformance_torequest_chat_image_base64() { +async fn test_vector_89_wireconformance_torequest_chat_image_base64() { let vector: Value = serde_json::from_str("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_91_wireconformance_torequest_chat_image_part() { +async fn test_vector_90_wireconformance_torequest_chat_image_part() { let vector: Value = serde_json::from_str("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_92_wireconformance_torequest_chat_multipart_content() { +async fn test_vector_91_wireconformance_torequest_chat_multipart_content() { let vector: Value = serde_json::from_str("{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_93_wireconformance_torequest_chat_simple() { +async fn test_vector_92_wireconformance_torequest_chat_simple() { let vector: Value = serde_json::from_str("{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_94_wireconformance_torequest_chat_single_text_optimized() { +async fn test_vector_93_wireconformance_torequest_chat_single_text_optimized() { let vector: Value = serde_json::from_str("{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_95_wireconformance_torequest_chat_with_options() { +async fn test_vector_94_wireconformance_torequest_chat_with_options() { let vector: Value = serde_json::from_str("{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_96_wireconformance_torequest_embedding_wire() { +async fn test_vector_95_wireconformance_torequest_embedding_wire() { let vector: Value = serde_json::from_str("{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_97_wireconformance_torequest_image_wire() { +async fn test_vector_96_wireconformance_torequest_image_wire() { let vector: Value = serde_json::from_str("{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_98_wireconformance_torequest_kind_to_json_type_mapping() { +async fn test_vector_97_wireconformance_torequest_kind_to_json_type_mapping() { let vector: Value = serde_json::from_str("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_99_wireconformance_torequest_options_additional_properties() { +async fn test_vector_98_wireconformance_torequest_options_additional_properties() { let vector: Value = serde_json::from_str("{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_100_wireconformance_torequest_options_max_completion_tokens() { +async fn test_vector_99_wireconformance_torequest_options_max_completion_tokens() { let vector: Value = serde_json::from_str("{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_101_wireconformance_torequest_options_stop_sequences() { +async fn test_vector_100_wireconformance_torequest_options_stop_sequences() { let vector: Value = serde_json::from_str("{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_102_wireconformance_torequest_responses_simple() { +async fn test_vector_101_wireconformance_torequest_responses_simple() { let vector: Value = serde_json::from_str("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_103_wireconformance_torequest_responses_structured_output() { +async fn test_vector_102_wireconformance_torequest_responses_structured_output() { let vector: Value = serde_json::from_str("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_104_wireconformance_torequest_responses_with_tools() { +async fn test_vector_103_wireconformance_torequest_responses_with_tools() { let vector: Value = serde_json::from_str("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_105_wireconformance_torequest_structured_output() { +async fn test_vector_104_wireconformance_torequest_structured_output() { let vector: Value = serde_json::from_str("{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_106_wireconformance_torequest_structured_output_nested_optional() { +async fn test_vector_105_wireconformance_torequest_structured_output_nested_optional() { let vector: Value = serde_json::from_str("{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_107_wireconformance_torequest_tools_bindings_stripped() { +async fn test_vector_106_wireconformance_torequest_tools_bindings_stripped() { let vector: Value = serde_json::from_str("{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_108_wireconformance_torequest_tools_function_wire() { +async fn test_vector_107_wireconformance_torequest_tools_function_wire() { let vector: Value = serde_json::from_str("{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_109_wireconformance_torequest_tools_null_when_empty() { +async fn test_vector_108_wireconformance_torequest_tools_null_when_empty() { let vector: Value = serde_json::from_str("{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_110_wireconformance_torequest_tools_strict_mode() { +async fn test_vector_109_wireconformance_torequest_tools_strict_mode() { let vector: Value = serde_json::from_str("{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; diff --git a/runtime/swift/prompty-model/Sources/PromptyModel/live_chat_conformance.swift b/runtime/swift/prompty-model/Sources/PromptyModel/live_chat_conformance.swift deleted file mode 100644 index 2083e2ca3..000000000 --- a/runtime/swift/prompty-model/Sources/PromptyModel/live_chat_conformance.swift +++ /dev/null @@ -1,8 +0,0 @@ -// -// Code generated by Typra emitter; DO NOT EDIT. - -import Foundation - -public protocol LiveChatConformance: Sendable { - func complete(input: Any) async throws -> Any -} diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift index 6f04efdd1..cbc614601 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift @@ -40,10 +40,7 @@ enum VectorAdapters { } static func waivers() -> [String: String] { - return [ - "LiveChatConformance.complete": - "drill: consumer-owned transport-seam probe (Python/TS only)" - ] + return [:] } static func doubles() -> Any? { diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift index c89f0b2bd..dbeb6e291 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift @@ -194,17 +194,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "DiscoveryConformance", operation: "mapModel", vector: vector, sync: false, seam: seam()) } - func testVector16LiveChatConformanceCompleteOpenaiChatDrill() async throws { - guard let vectorData = "{\"name\":\"openai_chat_drill\",\"stage\":\"drill\",\"provider\":\"openai\",\"input\":{\"provider\":\"openai\",\"model\":\"gpt-4o-mini\",\"apiKey\":{\"$env\":\"OPENAI_API_KEY\"},\"endpoint\":{\"$env\":\"OPENAI_BASE_URL\"},\"messages\":[{\"role\":\"user\",\"content\":\"Say hello in exactly one word.\"}],\"options\":{\"temperature\":0,\"maxOutputTokens\":16}},\"exchange\":{\"transport\":{\"baseUrl\":{\"$env\":\"OPENAI_BASE_URL\"}},\"cassette\":{\"$json\":\"cassettes/openai_chat_drill.json\"},\"planes\":[\"transport\",\"wire\",\"semantic\"]},\"expected\":{\"role\":\"assistant\",\"contentNonEmpty\":true,\"finishReasonInEnum\":true},\"operation\":\"complete\"}".data(using: .utf8), - let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] - else { - XCTFail("failed to parse embedded vector") - return - } - _ = await VectorRunner.runVector(contract: "LiveChatConformance", operation: "complete", vector: vector, sync: false, seam: seam()) - } - - func testVector17LoadConformanceLoadBasicLoad() async throws { + func testVector16LoadConformanceLoadBasicLoad() async throws { guard let vectorData = "{\"name\":\"basic_load\",\"description\":\"Load basic.prompty with env var resolution and verify all PromptAgent fields\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"basic-prompt\",\"description\":\"A basic prompt for testing\",\"metadata\":{\"authors\":[\"testauthor\"]},\"model\":{\"id\":\"gpt-4\",\"provider\":\"openai\",\"apiType\":\"chat\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"},\"options\":{\"temperature\":0.7,\"maxOutputTokens\":1000}},\"inputs\":[{\"name\":\"firstName\",\"kind\":\"string\",\"default\":\"Jane\"},{\"name\":\"lastName\",\"kind\":\"string\",\"default\":\"Doe\"},{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What is the meaning of life?\"}],\"template\":{\"format\":{\"kind\":\"jinja2\"},\"parser\":{\"kind\":\"prompty\"}},\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -214,7 +204,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector18LoadConformanceLoadConnectionTypesLoad() async throws { + func testVector17LoadConformanceLoadConnectionTypesLoad() async throws { guard let vectorData = "{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -224,7 +214,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector19LoadConformanceLoadEmbeddingLoad() async throws { + func testVector18LoadConformanceLoadEmbeddingLoad() async throws { guard let vectorData = "{\"name\":\"embedding_load\",\"description\":\"Load embedding.prompty and verify apiType=embedding with correct model id\",\"stage\":\"load\",\"input\":{\"fixture\":\"embedding.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"embedding\",\"model\":{\"id\":\"text-embedding-3-small\",\"provider\":\"openai\",\"apiType\":\"embedding\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://test.openai.com\"}},\"inputs\":[{\"name\":\"text\",\"kind\":\"string\",\"default\":\"Hello world\"}],\"instructions\":\"{{text}}\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -234,7 +224,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector20LoadConformanceLoadEmptyFrontmatterBodyOnly() async throws { + func testVector19LoadConformanceLoadEmptyFrontmatterBodyOnly() async throws { guard let vectorData = "{\"name\":\"empty_frontmatter_body_only\",\"description\":\"Empty frontmatter with body \u{2014} instructions extracted, defaults applied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"empty-fm\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"empty-fm\",\"model\":null,\"inputs\":null,\"tools\":null},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -244,7 +234,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector21LoadConformanceLoadEnvDefault() async throws { + func testVector20LoadConformanceLoadEnvDefault() async throws { guard let vectorData = "{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -254,7 +244,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector22LoadConformanceLoadEnvMissingError() async throws { + func testVector21LoadConformanceLoadEnvMissingError() async throws { guard let vectorData = "{\"name\":\"env_missing_error\",\"description\":\"${env:NONEXISTENT} with no default and var not set raises an error\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-error-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:NONEXISTENT}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"env_var_not_set\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -264,7 +254,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector23LoadConformanceLoadEnvResolution() async throws { + func testVector22LoadConformanceLoadEnvResolution() async throws { guard let vectorData = "{\"name\":\"env_resolution\",\"description\":\"${env:MY_VAR} resolves to the environment variable value when set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MY_VAR}\"}}},\"env\":{\"MY_VAR\":\"hello\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"hello\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -274,7 +264,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector24LoadConformanceLoadFileReferenceInTreeAllowed() async throws { + func testVector23LoadConformanceLoadFileReferenceInTreeAllowed() async throws { guard let vectorData = "{\"name\":\"file_reference_in_tree_allowed\",\"description\":\"A ${file:sub/data.json} reference that stays within the prompt file's directory tree resolves to the parsed content (containment control alongside the traversal-reject vector)\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-in-tree-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:sub/data.json}\"}},\"files\":{\"sub/data.json\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\",\"apiKey\":\"ok\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://in-tree.example.com\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -284,7 +274,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector25LoadConformanceLoadFileReferenceTraversalRejected() async throws { + func testVector24LoadConformanceLoadFileReferenceTraversalRejected() async throws { guard let vectorData = "{\"name\":\"file_reference_traversal_rejected\",\"description\":\"A ${file:../secret.json} reference that escapes the prompt file's directory via `..` is rejected even though the target file exists, preventing arbitrary file disclosure (GHSA-7pfc-5v9r-j299). Uses forward-slash separators so the vector holds on every OS.\",\"stage\":\"load\",\"input\":{\"agent_subdir\":\"app\",\"frontmatter\":{\"name\":\"file-traversal-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:../secret.json}\"}},\"files\":{\"../secret.json\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"leaked\"}}},\"expectedError\":{\"kind\":\"file_reference\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -294,7 +284,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector26LoadConformanceLoadFileResolution() async throws { + func testVector25LoadConformanceLoadFileResolution() async throws { guard let vectorData = "{\"name\":\"file_resolution\",\"description\":\"${file:shared_connection.json} resolves to the parsed JSON content of the referenced file\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"file-ref-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:shared_connection.json}\"}},\"env\":{\"OPENAI_API_KEY\":\"shared-key\"},\"files\":{\"shared_connection.json\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\",\"apiKey\":\"shared-key\"}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://shared.example.com\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -304,7 +294,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector27LoadConformanceLoadImageApitypeLoad() async throws { + func testVector26LoadConformanceLoadImageApitypeLoad() async throws { guard let vectorData = "{\"name\":\"image_apitype_load\",\"description\":\"An image-generation prompt loads with apiType image and passthrough additionalProperties model options\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}}},\"expected\":{\"kind\":\"prompt\",\"name\":\"image-gen\",\"model\":{\"id\":\"dall-e-3\",\"provider\":\"openai\",\"apiType\":\"image\",\"options\":{\"additionalProperties\":{\"size\":\"1024x1024\",\"quality\":\"standard\"}}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -314,7 +304,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector28LoadConformanceLoadInputScalarShorthand() async throws { + func testVector27LoadConformanceLoadInputScalarShorthand() async throws { guard let vectorData = "{\"name\":\"input_scalar_shorthand\",\"description\":\"Scalar input values create typed Property with default set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"scalar-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]}},\"expected\":{\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"default\":\"science\"},{\"name\":\"count\",\"kind\":\"integer\",\"default\":5}]},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -324,7 +314,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector29LoadConformanceLoadInputValidationDefaultFill() async throws { + func testVector28LoadConformanceLoadInputValidationDefaultFill() async throws { guard let vectorData = "{\"name\":\"input_validation_default_fill\",\"description\":\"An input with a default value is filled in when no value is provided\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"default-fill-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"name\",\"kind\":\"string\",\"default\":\"world\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{\"name\":\"world\"}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -334,7 +324,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector30LoadConformanceLoadInputValidationExampleNotUsed() async throws { + func testVector29LoadConformanceLoadInputValidationExampleNotUsed() async throws { guard let vectorData = "{\"name\":\"input_validation_example_not_used\",\"description\":\"An input with example='sample' but no default must NOT use the example at runtime\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"example-not-used-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"topic\",\"kind\":\"string\",\"required\":false,\"example\":\"sample\"}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -344,7 +334,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector31LoadConformanceLoadInputValidationOptionalOmit() async throws { + func testVector30LoadConformanceLoadInputValidationOptionalOmit() async throws { guard let vectorData = "{\"name\":\"input_validation_optional_omit\",\"description\":\"A non-required input with no default and no value is omitted (not an error)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"optional-omit-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"nickname\",\"kind\":\"string\",\"required\":false}]},\"inputs\":{}},\"expected\":{\"validated_inputs\":{}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -354,7 +344,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector32LoadConformanceLoadInputValidationRequired() async throws { + func testVector31LoadConformanceLoadInputValidationRequired() async throws { guard let vectorData = "{\"name\":\"input_validation_required\",\"description\":\"A required input with no default and no value provided raises ValueError\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"required-input-test\",\"model\":\"gpt-4\",\"inputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},\"inputs\":{}},\"expectedError\":{\"kind\":\"missing_required_input\",\"field\":\"city\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -364,7 +354,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector33LoadConformanceLoadInstructionsFromBody() async throws { + func testVector32LoadConformanceLoadInstructionsFromBody() async throws { guard let vectorData = "{\"name\":\"instructions_from_body\",\"description\":\"The markdown body after the closing --- becomes the instructions field\",\"stage\":\"load\",\"input\":{\"fixture\":\"basic.prompty\",\"env\":{\"OPENAI_ENDPOINT\":\"https://test.openai.com\",\"OPENAI_API_KEY\":\"sk-test123\"}},\"expected\":{\"instructions\":\"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -374,7 +364,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector34LoadConformanceLoadInvalidFrontmatterError() async throws { + func testVector33LoadConformanceLoadInvalidFrontmatterError() async throws { guard let vectorData = "{\"name\":\"invalid_frontmatter_error\",\"description\":\"Malformed YAML in frontmatter raises a parse error\",\"stage\":\"load\",\"input\":{\"frontmatter_raw\":\"---\\nname: [invalid\\n---\\nHello\"},\"expectedError\":{\"kind\":\"invalid_frontmatter\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -384,7 +374,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector35LoadConformanceLoadKindAlwaysPrompt() async throws { + func testVector34LoadConformanceLoadKindAlwaysPrompt() async throws { guard let vectorData = "{\"name\":\"kind_always_prompt\",\"description\":\"The loader always injects kind='prompt' \u{2014} .prompty files never specify kind themselves\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -394,7 +384,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector36LoadConformanceLoadMinimalLoad() async throws { + func testVector35LoadConformanceLoadMinimalLoad() async throws { guard let vectorData = "{\"name\":\"minimal_load\",\"description\":\"Load minimal.prompty with model shorthand and bare-minimum frontmatter\",\"stage\":\"load\",\"input\":{\"fixture\":\"minimal.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"minimal\",\"model\":{\"id\":\"gpt-4\"},\"instructions\":\"system:\\nHello world.\",\"inputs\":null,\"outputs\":null,\"tools\":null},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -404,7 +394,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector37LoadConformanceLoadMissingFileError() async throws { + func testVector36LoadConformanceLoadMissingFileError() async throws { guard let vectorData = "{\"name\":\"missing_file_error\",\"description\":\"Loading a nonexistent .prompty file raises FileNotFoundError\",\"stage\":\"load\",\"input\":{\"fixture\":\"nonexistent.prompty\"},\"expectedError\":{\"kind\":\"file_not_found\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -414,7 +404,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector38LoadConformanceLoadModelShorthand() async throws { + func testVector37LoadConformanceLoadModelShorthand() async throws { guard let vectorData = "{\"name\":\"model_shorthand\",\"description\":\"String shorthand 'model: gpt-4o' expands to a Model object with id set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"test\",\"model\":\"gpt-4o\"}},\"expected\":{\"kind\":\"prompt\",\"name\":\"test\",\"model\":{\"id\":\"gpt-4o\"}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -424,7 +414,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector39LoadConformanceLoadStructuredOutputsLoad() async throws { + func testVector38LoadConformanceLoadStructuredOutputsLoad() async throws { guard let vectorData = "{\"name\":\"structured_outputs_load\",\"description\":\"Outputs load as a list of properties preserving kind, description, and required flags\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"structured-output\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]}},\"expected\":{\"kind\":\"prompt\",\"name\":\"structured-output\",\"outputs\":[{\"name\":\"temperature\",\"kind\":\"integer\",\"description\":\"Temperature in Fahrenheit\",\"required\":true},{\"name\":\"condition\",\"kind\":\"string\",\"description\":\"Weather condition\",\"required\":true},{\"name\":\"city\",\"kind\":\"string\",\"description\":\"City name\",\"required\":true}]},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -434,7 +424,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector40LoadConformanceLoadTemplateStringInvalid() async throws { + func testVector39LoadConformanceLoadTemplateStringInvalid() async throws { guard let vectorData = "{\"name\":\"template_string_invalid\",\"description\":\"Template as a bare string is not valid v2 \u{2014} must be an object with format/parser\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"shorthand-test\",\"model\":{\"id\":\"gpt-4\"},\"template\":\"jinja2\"}},\"expectedError\":{\"kind\":\"invalid_template\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -444,7 +434,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector41LoadConformanceLoadToolsCustomLoad() async throws { + func testVector40LoadConformanceLoadToolsCustomLoad() async throws { guard let vectorData = "{\"name\":\"tools_custom_load\",\"description\":\"Load a prompty with an unknown tool kind \u{2014} falls through to CustomTool\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"custom-tool-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://custom.example.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"my_tool\",\"kind\":\"my_provider\"}]},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -454,7 +444,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector42LoadConformanceLoadToolsFunctionLoad() async throws { + func testVector41LoadConformanceLoadToolsFunctionLoad() async throws { guard let vectorData = "{\"name\":\"tools_function_load\",\"description\":\"Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings\",\"stage\":\"load\",\"input\":{\"fixture\":\"tools_function.prompty\"},\"expected\":{\"kind\":\"prompt\",\"name\":\"function-tools\",\"model\":{\"id\":\"gpt-4\",\"apiType\":\"chat\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather in a given location\",\"strict\":true,\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"description\":\"The city and state, e.g. San Francisco, CA\"},{\"name\":\"unit\",\"kind\":\"string\",\"description\":\"Temperature unit\",\"enumValues\":[\"celsius\",\"fahrenheit\"]}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"inputs\":[{\"name\":\"question\",\"kind\":\"string\",\"default\":\"What's the weather in Paris?\"},{\"name\":\"preferred_unit\",\"kind\":\"string\",\"default\":\"celsius\"}],\"instructions\":\"system:\\nYou are a helpful assistant with access to tools.\\n\\nuser:\\n{{question}}\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -464,7 +454,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector43LoadConformanceLoadToolsMcpLoad() async throws { + func testVector42LoadConformanceLoadToolsMcpLoad() async throws { guard let vectorData = "{\"name\":\"tools_mcp_load\",\"description\":\"Load a prompty with an MCP tool \u{2014} kind, serverName, connection preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"mcp-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-mcp\"}}]}},\"expected\":{\"tools\":[{\"name\":\"filesystem\",\"kind\":\"mcp\",\"serverName\":\"fs-server\"}]},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -474,7 +464,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector44LoadConformanceLoadToolsOpenapiLoad() async throws { + func testVector43LoadConformanceLoadToolsOpenapiLoad() async throws { guard let vectorData = "{\"name\":\"tools_openapi_load\",\"description\":\"Load a prompty with an OpenAPI tool \u{2014} specification path preserved\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"openapi-test\",\"model\":\"gpt-4\",\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://api.weather.com\"}}]}},\"expected\":{\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\"}]},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -484,7 +474,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector45TurnConformanceReplayMaxIterations() async throws { + func testVector44TurnConformanceReplayMaxIterations() async throws { guard let vectorData = "{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -494,7 +484,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector46TurnConformanceReplayNoTool() async throws { + func testVector45TurnConformanceReplayNoTool() async throws { guard let vectorData = "{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -504,7 +494,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector47TurnConformanceReplayPermissionDenied() async throws { + func testVector46TurnConformanceReplayPermissionDenied() async throws { guard let vectorData = "{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -514,7 +504,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector48TurnConformanceReplayToolFailure() async throws { + func testVector47TurnConformanceReplayToolFailure() async throws { guard let vectorData = "{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -524,7 +514,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector49TurnConformanceReplayToolSuccess() async throws { + func testVector48TurnConformanceReplayToolSuccess() async throws { guard let vectorData = "{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -534,7 +524,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector50TurnConformanceRunAssistantToolCallsMetadata() async throws { + func testVector49TurnConformanceRunAssistantToolCallsMetadata() async throws { guard let vectorData = "{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -544,7 +534,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector51TurnConformanceRunAsyncToolFunction() async throws { + func testVector50TurnConformanceRunAsyncToolFunction() async throws { guard let vectorData = "{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -554,7 +544,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector52TurnConformanceRunBindingsInjected() async throws { + func testVector51TurnConformanceRunBindingsInjected() async throws { guard let vectorData = "{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -564,7 +554,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector53TurnConformanceRunCancellationBeforeLlm() async throws { + func testVector52TurnConformanceRunCancellationBeforeLlm() async throws { guard let vectorData = "{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -574,7 +564,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector54TurnConformanceRunCancellationBetweenIterations() async throws { + func testVector53TurnConformanceRunCancellationBetweenIterations() async throws { guard let vectorData = "{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -584,7 +574,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector55TurnConformanceRunCancellationBetweenTools() async throws { + func testVector54TurnConformanceRunCancellationBetweenTools() async throws { guard let vectorData = "{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -594,7 +584,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector56TurnConformanceRunContextNoTrimWhenFits() async throws { + func testVector55TurnConformanceRunContextNoTrimWhenFits() async throws { guard let vectorData = "{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -604,7 +594,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector57TurnConformanceRunContextPreservesSystemMessages() async throws { + func testVector56TurnConformanceRunContextPreservesSystemMessages() async throws { guard let vectorData = "{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -614,7 +604,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector58TurnConformanceRunContextTrimBasic() async throws { + func testVector57TurnConformanceRunContextTrimBasic() async throws { guard let vectorData = "{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -624,7 +614,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector59TurnConformanceRunEmptyToolResult() async throws { + func testVector58TurnConformanceRunEmptyToolResult() async throws { guard let vectorData = "{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -634,7 +624,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector60TurnConformanceRunEventsBasicToolLoop() async throws { + func testVector59TurnConformanceRunEventsBasicToolLoop() async throws { guard let vectorData = "{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -644,7 +634,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector61TurnConformanceRunEventsErrorLogged() async throws { + func testVector60TurnConformanceRunEventsErrorLogged() async throws { guard let vectorData = "{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -654,7 +644,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector62TurnConformanceRunEventsNoTools() async throws { + func testVector61TurnConformanceRunEventsNoTools() async throws { guard let vectorData = "{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -664,7 +654,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector63TurnConformanceRunGuardrailAllPass() async throws { + func testVector62TurnConformanceRunGuardrailAllPass() async throws { guard let vectorData = "{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -674,7 +664,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector64TurnConformanceRunGuardrailInputDeny() async throws { + func testVector63TurnConformanceRunGuardrailInputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -684,7 +674,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector65TurnConformanceRunGuardrailOutputDeny() async throws { + func testVector64TurnConformanceRunGuardrailOutputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -694,7 +684,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector66TurnConformanceRunGuardrailToolDeny() async throws { + func testVector65TurnConformanceRunGuardrailToolDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -704,7 +694,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector67TurnConformanceRunMaxIterationsExceeded() async throws { + func testVector66TurnConformanceRunMaxIterationsExceeded() async throws { guard let vectorData = "{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -714,7 +704,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector68TurnConformanceRunMultiTurnToolCalls() async throws { + func testVector67TurnConformanceRunMultiTurnToolCalls() async throws { guard let vectorData = "{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -724,7 +714,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector69TurnConformanceRunMultipleToolCallsSingleTurn() async throws { + func testVector68TurnConformanceRunMultipleToolCallsSingleTurn() async throws { guard let vectorData = "{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -734,7 +724,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector70TurnConformanceRunNoToolCalls() async throws { + func testVector69TurnConformanceRunNoToolCalls() async throws { guard let vectorData = "{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -744,7 +734,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector71TurnConformanceRunParallelToolsBasic() async throws { + func testVector70TurnConformanceRunParallelToolsBasic() async throws { guard let vectorData = "{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -754,7 +744,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector72TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { + func testVector71TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"rust_expected_error\":\"parallel_tool_calls=true is not supported by the canonical Rust engine\",\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -764,7 +754,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector73TurnConformanceRunSingleToolCall() async throws { + func testVector72TurnConformanceRunSingleToolCall() async throws { guard let vectorData = "{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -774,7 +764,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector74TurnConformanceRunSteeringInjectMessage() async throws { + func testVector73TurnConformanceRunSteeringInjectMessage() async throws { guard let vectorData = "{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -784,7 +774,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector75TurnConformanceRunSteeringMultipleMessages() async throws { + func testVector74TurnConformanceRunSteeringMultipleMessages() async throws { guard let vectorData = "{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -794,7 +784,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector76TurnConformanceRunToolNotRegisteredError() async throws { + func testVector75TurnConformanceRunToolNotRegisteredError() async throws { guard let vectorData = "{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -804,7 +794,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector77TurnConformanceRunToolResultMessageFormat() async throws { + func testVector76TurnConformanceRunToolResultMessageFormat() async throws { guard let vectorData = "{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -814,7 +804,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector78TurnConformanceRunTurnCancelBeforeContext() async throws { + func testVector77TurnConformanceRunTurnCancelBeforeContext() async throws { guard let vectorData = "{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -824,7 +814,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector79TurnConformanceRunTurnDelegatedProviderState() async throws { + func testVector78TurnConformanceRunTurnDelegatedProviderState() async throws { guard let vectorData = "{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -834,7 +824,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector80TurnConformanceRunTurnFinalOutput() async throws { + func testVector79TurnConformanceRunTurnFinalOutput() async throws { guard let vectorData = "{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -844,7 +834,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector81TurnConformanceRunTurnOrderedToolRound() async throws { + func testVector80TurnConformanceRunTurnOrderedToolRound() async throws { guard let vectorData = "{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -854,7 +844,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector82TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { + func testVector81TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { guard let vectorData = "{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -864,7 +854,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector83WireConformanceToRequestAnthropicImageFormat() async throws { + func testVector82WireConformanceToRequestAnthropicImageFormat() async throws { guard let vectorData = "{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -874,7 +864,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector84WireConformanceToRequestAnthropicMaxTokensRequired() async throws { + func testVector83WireConformanceToRequestAnthropicMaxTokensRequired() async throws { guard let vectorData = "{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -884,7 +874,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector85WireConformanceToRequestAnthropicOptions() async throws { + func testVector84WireConformanceToRequestAnthropicOptions() async throws { guard let vectorData = "{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -894,7 +884,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector86WireConformanceToRequestAnthropicSystemSeparate() async throws { + func testVector85WireConformanceToRequestAnthropicSystemSeparate() async throws { guard let vectorData = "{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -904,7 +894,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector87WireConformanceToRequestAnthropicToolWire() async throws { + func testVector86WireConformanceToRequestAnthropicToolWire() async throws { guard let vectorData = "{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -914,7 +904,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector88WireConformanceToRequestChatAudioMp3() async throws { + func testVector87WireConformanceToRequestChatAudioMp3() async throws { guard let vectorData = "{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -924,7 +914,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector89WireConformanceToRequestChatAudioPart() async throws { + func testVector88WireConformanceToRequestChatAudioPart() async throws { guard let vectorData = "{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -934,7 +924,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector90WireConformanceToRequestChatImageBase64() async throws { + func testVector89WireConformanceToRequestChatImageBase64() async throws { guard let vectorData = "{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -944,7 +934,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector91WireConformanceToRequestChatImagePart() async throws { + func testVector90WireConformanceToRequestChatImagePart() async throws { guard let vectorData = "{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -954,7 +944,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector92WireConformanceToRequestChatMultipartContent() async throws { + func testVector91WireConformanceToRequestChatMultipartContent() async throws { guard let vectorData = "{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -964,7 +954,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector93WireConformanceToRequestChatSimple() async throws { + func testVector92WireConformanceToRequestChatSimple() async throws { guard let vectorData = "{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -974,7 +964,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector94WireConformanceToRequestChatSingleTextOptimized() async throws { + func testVector93WireConformanceToRequestChatSingleTextOptimized() async throws { guard let vectorData = "{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -984,7 +974,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector95WireConformanceToRequestChatWithOptions() async throws { + func testVector94WireConformanceToRequestChatWithOptions() async throws { guard let vectorData = "{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -994,7 +984,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector96WireConformanceToRequestEmbeddingWire() async throws { + func testVector95WireConformanceToRequestEmbeddingWire() async throws { guard let vectorData = "{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1004,7 +994,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector97WireConformanceToRequestImageWire() async throws { + func testVector96WireConformanceToRequestImageWire() async throws { guard let vectorData = "{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1014,7 +1004,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector98WireConformanceToRequestKindToJsonTypeMapping() async throws { + func testVector97WireConformanceToRequestKindToJsonTypeMapping() async throws { guard let vectorData = "{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1024,7 +1014,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector99WireConformanceToRequestOptionsAdditionalProperties() async throws { + func testVector98WireConformanceToRequestOptionsAdditionalProperties() async throws { guard let vectorData = "{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1034,7 +1024,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector100WireConformanceToRequestOptionsMaxCompletionTokens() async throws { + func testVector99WireConformanceToRequestOptionsMaxCompletionTokens() async throws { guard let vectorData = "{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1044,7 +1034,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector101WireConformanceToRequestOptionsStopSequences() async throws { + func testVector100WireConformanceToRequestOptionsStopSequences() async throws { guard let vectorData = "{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1054,7 +1044,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector102WireConformanceToRequestResponsesSimple() async throws { + func testVector101WireConformanceToRequestResponsesSimple() async throws { guard let vectorData = "{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1064,7 +1054,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector103WireConformanceToRequestResponsesStructuredOutput() async throws { + func testVector102WireConformanceToRequestResponsesStructuredOutput() async throws { guard let vectorData = "{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1074,7 +1064,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector104WireConformanceToRequestResponsesWithTools() async throws { + func testVector103WireConformanceToRequestResponsesWithTools() async throws { guard let vectorData = "{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1084,7 +1074,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector105WireConformanceToRequestStructuredOutput() async throws { + func testVector104WireConformanceToRequestStructuredOutput() async throws { guard let vectorData = "{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1094,7 +1084,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector106WireConformanceToRequestStructuredOutputNestedOptional() async throws { + func testVector105WireConformanceToRequestStructuredOutputNestedOptional() async throws { guard let vectorData = "{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1104,7 +1094,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector107WireConformanceToRequestToolsBindingsStripped() async throws { + func testVector106WireConformanceToRequestToolsBindingsStripped() async throws { guard let vectorData = "{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1114,7 +1104,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector108WireConformanceToRequestToolsFunctionWire() async throws { + func testVector107WireConformanceToRequestToolsFunctionWire() async throws { guard let vectorData = "{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1124,7 +1114,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector109WireConformanceToRequestToolsNullWhenEmpty() async throws { + func testVector108WireConformanceToRequestToolsNullWhenEmpty() async throws { guard let vectorData = "{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1134,7 +1124,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector110WireConformanceToRequestToolsStrictMode() async throws { + func testVector109WireConformanceToRequestToolsStrictMode() async throws { guard let vectorData = "{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { diff --git a/runtime/typescript/packages/core/src/model/index.ts b/runtime/typescript/packages/core/src/model/index.ts index 4e1bf721d..5ca5aa4f0 100644 --- a/runtime/typescript/packages/core/src/model/index.ts +++ b/runtime/typescript/packages/core/src/model/index.ts @@ -14,7 +14,6 @@ export type { EventJournalWriter } from "./event-journal-writer"; export type { EventSink } from "./event-sink"; export type { Executor } from "./executor"; export type { HostToolExecutor } from "./host-tool-executor"; -export type { LiveChatConformance } from "./live-chat-conformance"; export type { LoadConformance } from "./load-conformance"; export type { ModelLister } from "./model-lister"; export type { Parser } from "./parser"; diff --git a/runtime/typescript/packages/core/src/model/live-chat-conformance.ts b/runtime/typescript/packages/core/src/model/live-chat-conformance.ts deleted file mode 100644 index 365d29aab..000000000 --- a/runtime/typescript/packages/core/src/model/live-chat-conformance.ts +++ /dev/null @@ -1,15 +0,0 @@ -// -// Copyright (c) Microsoft. All rights reserved. -// WARNING: This is an auto-generated file. DO NOT EDIT THIS FILE DIRECTLY. - -/** Conformance-only seam hosting the "service drill" vectors: a conformance -vector that crosses the REAL Executor->Processor transport seam instead of -stopping at the SDK edge. A drill runs against a live provider when -credentials are present and REPLAYS a recorded cassette when they are not; -the live<->replay parity is the gate. - -Like every other conformance seam this interface exists purely to carry */ -export interface LiveChatConformance { - /** Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes */ - complete(input: unknown): Promise; -} diff --git a/runtime/typescript/packages/core/tests/model/cassettes/openai_chat_drill.json b/runtime/typescript/packages/core/tests/model/cassettes/openai_chat_drill.json deleted file mode 100644 index 8afdc52e0..000000000 --- a/runtime/typescript/packages/core/tests/model/cassettes/openai_chat_drill.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "_comment": "Service-drill cassette for LiveChatConformance.complete/openai_chat_drill. Consumer-owned roster: NOT modeled in TypeSpec/Typra. Regenerate with `uv run python tests/model/record_cassette.py`. `request.body` is the wire plane the drill adapter asserts; `response.body` is replayed as the seam bytes.", - "recordedAt": "2026-09-04T22:41:20Z", - "request": { - "body": { - "max_completion_tokens": 16, - "messages": [ - { - "content": "Say hello in exactly one word.", - "role": "user" - } - ], - "model": "gpt-4o-mini", - "temperature": 0 - }, - "method": "POST", - "path": "/chat/completions" - }, - "response": { - "body": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "logprobs": null, - "message": { - "content": "Hello", - "role": "assistant" - } - } - ], - "created": 1700000000, - "id": "chatcmpl-drill-openai-0001", - "model": "gpt-4o-mini", - "object": "chat.completion", - "usage": { - "completion_tokens": 1, - "prompt_tokens": 12, - "total_tokens": 13 - } - }, - "status": 200 - } -} diff --git a/runtime/typescript/packages/core/tests/model/drill-replay.ts b/runtime/typescript/packages/core/tests/model/drill-replay.ts deleted file mode 100644 index 05357bdda..000000000 --- a/runtime/typescript/packages/core/tests/model/drill-replay.ts +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Local cassette-replay HTTP transport for the service-drill vector. - * - * The drill binds the Executor->Processor transport seam via **base-URL - * redirect**: the provider SDK is pointed at a local `node:http` server that - * serves a recorded response and records the inbound request. This is the - * consumer-owned "roster" half of the drill -- deliberately NOT modeled in - * TypeSpec/Typra, since cassette bytes and provider-SDK transport binding stay - * runtime-owned. It is the TypeScript counterpart of the Python runtime's - * `tests/model/_drill_replay.py`. - * - * No production code changes: the executor's own connection `endpoint` is set - * to `server.baseUrl` and its OpenAI SDK POSTs there exactly as it would to the - * real base URL. - */ - -import { createServer, type IncomingMessage, type Server } from "node:http"; -import type { AddressInfo } from "node:net"; - -export type CapturedRequest = { - method: string; - path: string; - body: unknown; -}; - -/** - * A background HTTP server that replays one response and records requests. - * - * `baseUrl` is the redirect target to hand the provider SDK; `lastRequest` is - * the most recently captured outbound request (the wire plane's observed - * value). Use `start()`/`stop()` around the transport-bound invocation. - */ -export class CassetteReplayServer { - readonly captured: CapturedRequest[] = []; - private server: Server | null = null; - - constructor( - private readonly responseBody: unknown, - private readonly responseStatus: number = 200, - ) {} - - async start(): Promise { - const payload = JSON.stringify(this.responseBody); - this.server = createServer((req: IncomingMessage, res) => { - const chunks: Buffer[] = []; - req.on("data", (chunk: Buffer) => chunks.push(chunk)); - req.on("end", () => { - const raw = Buffer.concat(chunks).toString("utf8"); - let body: unknown = {}; - if (raw) { - try { - body = JSON.parse(raw); - } catch { - body = { __raw__: raw }; - } - } - this.captured.push({ - method: req.method ?? "POST", - path: req.url ?? "/", - body, - }); - res.writeHead(this.responseStatus, { - "Content-Type": "application/json", - "Content-Length": Buffer.byteLength(payload).toString(), - }); - res.end(payload); - }); - }); - await new Promise((resolveListen) => { - this.server!.listen(0, "127.0.0.1", () => resolveListen()); - }); - } - - async stop(): Promise { - if (!this.server) return; - await new Promise((resolveClose) => { - this.server!.close(() => resolveClose()); - }); - this.server = null; - } - - get baseUrl(): string { - if (!this.server) throw new Error("replay server not started"); - const { address, port } = this.server.address() as AddressInfo; - const host = address === "::" ? "127.0.0.1" : address; - return `http://${host}:${port}`; - } - - get lastRequest(): CapturedRequest | null { - return this.captured.length > 0 - ? this.captured[this.captured.length - 1] - : null; - } -} diff --git a/runtime/typescript/packages/core/tests/model/vector-adapters.ts b/runtime/typescript/packages/core/tests/model/vector-adapters.ts index bf9bfe4e8..5a75d703d 100644 --- a/runtime/typescript/packages/core/tests/model/vector-adapters.ts +++ b/runtime/typescript/packages/core/tests/model/vector-adapters.ts @@ -81,23 +81,11 @@ import { defaultSaveContext } from "../../src/core/loader.js"; import { TurnOptions } from "../../src/model/index.js"; import { enrich, mapModel } from "../../src/model/discovery.js"; import { ModelInfo } from "../../src/model/contracts/models/model-info.js"; -// Drill transport bind: the OpenAI executor checks `connection instanceof -// ApiKeyConnection` against the built `@prompty/core` package (dist), whose class -// identity differs from the `../../src` core imported above. The drill agent must -// therefore be constructed from the dist package so the executor recognizes its -// key connection. This cross-core seam is a per-runtime roster detail, not an -// emitter concern. -import { - Agent as DrillAgent, - LoadContext as DrillLoadContext, - Message as DrillMessage, -} from "@prompty/core"; import { buildChatArgs as openaiBuildChatArgs, buildEmbeddingArgs as openaiBuildEmbeddingArgs, buildImageArgs as openaiBuildImageArgs, buildResponsesArgs as openaiBuildResponsesArgs, - OpenAIExecutor, OpenAIProcessor, processResponse as openaiProcessResponse, } from "@prompty/openai"; @@ -105,7 +93,6 @@ import { AnthropicProcessor, buildChatArgs as anthropicBuildChatArgs, } from "@prompty/anthropic"; -import { CassetteReplayServer } from "./drill-replay.js"; // The pipeline drives renderer/parser lookups through the registry; register the // built-ins once so the render/parse adapters exercise the real runtime path. @@ -1071,249 +1058,6 @@ async function runTurnInvoke( }; } -// --------------------------------------------------------------------------- -// LiveChatConformance.complete -- the service-drill adapter -// --------------------------------------------------------------------------- -// -// A "drill" is a conformance vector that crosses the REAL Executor transport -// seam. Unlike the pure conformance vectors that stop at the SDK edge, the drill -// binds the seam via base-URL redirect -- to a local `CassetteReplayServer` -// (replay mode) or the real base URL (live mode) -- and asserts three planes: -// -// 1. transport -- the seam was reached and returned bytes (executor got a -// response, and the replay server captured an inbound request). -// 2. wire -- the outbound provider request body matches the cassette's -// recorded request body (canonical equality; throw on mismatch). -// 3. semantic -- normalized executor output == the vector `expected`. This -// plane is asserted by the RUNNER via the registered `normalize`, so the -// drill's semantic plane IS a conformance vector over the executor. -// -// In live mode the adapter additionally asserts live-semantic == replay-semantic -// (parity). `requires` is deliberately OMITTED from the drill vector -- the -// `requires` guard is all-or-nothing and would skip the WHOLE vector when creds -// are absent, defeating the replay plane. Instead the adapter self-gates the -// live plane on credential presence and always runs replay. -// -// Cassette bytes and the SDK transport binding are consumer-owned "roster" -- by -// design NOT modeled in TypeSpec/Typra. This is the TypeScript counterpart of -// the Python runtime's drill adapter in `tests/model/vector_adapters.py`. - -const DRILL_REPLAY_API_KEY = "sk-drill-replay-placeholder"; -const DRILL_FINISH_REASONS = new Set([ - "stop", - "length", - "tool_calls", - "content_filter", - "function_call", -]); - -/** Order-insensitive canonical JSON for wire/parity equality checks. */ -function drillCanonical(value: unknown): string { - const sort = (node: unknown): unknown => { - if (Array.isArray(node)) return node.map(sort); - if (node !== null && typeof node === "object") { - const src = node as Record; - const out: Record = {}; - for (const key of Object.keys(src).sort()) out[key] = sort(src[key]); - return out; - } - return node; - }; - return JSON.stringify(sort(value)); -} - -/** - * Construct a chat `Agent` bound to a specific transport `endpoint`. - * - * The drill binds the transport seam via base-URL redirect, so `endpoint` and - * `apiKey` are supplied by the caller (a local cassette-replay server in replay - * mode, the real base URL + key in live mode) rather than read from the vector - * input. When `endpoint` is falsy the connection omits it so the SDK falls back - * to the provider's default base URL (live mode without an override). - */ -function drillBuildAgent( - input: any, - opts: { endpoint?: string | null; apiKey?: string | null }, -): DrillAgent { - const provider = (input.provider ?? "openai").toLowerCase(); - const connection: Record = { kind: "key" }; - if (opts.apiKey) connection.apiKey = opts.apiKey; - if (opts.endpoint) connection.endpoint = opts.endpoint; - - const model: Record = { - id: input.model ?? "gpt-4o-mini", - provider, - apiType: "chat", - connection, - }; - if (input.options) model.options = input.options; - - return DrillAgent.load( - { name: "drill-chat-vector", model }, - new DrillLoadContext(), - ); -} - -function drillBuildMessages(input: any): DrillMessage[] { - return ((input.messages ?? []) as any[]).map( - (spec) => - new DrillMessage({ - role: spec.role ?? "user", - parts: [{ kind: "text", value: spec.content ?? "" }], - }), - ); -} - -/** - * Drive the real executor with its transport bound to `endpoint`. Returns the - * raw provider response. Shared by the replay path (`endpoint` = a - * `CassetteReplayServer` base URL) and the live path (`endpoint` = the real - * base URL or null for the SDK default). - */ -async function drillExecute( - input: any, - opts: { endpoint?: string | null; apiKey?: string | null }, -): Promise { - const agent = drillBuildAgent(input, opts); - const messages = drillBuildMessages(input); - return new OpenAIExecutor().execute(agent, messages); -} - -/** Reduce a raw provider chat response to `{ role, content, finish }`. */ -function drillExtractChatStructure(observed: any): { - role: unknown; - content: unknown; - finish: unknown; -} { - const choices = observed?.choices; - if (Array.isArray(choices) && choices.length > 0) { - const choice = choices[0]; - const message = choice?.message; - return { - role: message?.role, - content: message?.content, - finish: choice?.finish_reason, - }; - } - return { - role: observed?.role, - content: observed?.content, - finish: observed?.stop_reason, - }; -} - -/** Project a raw chat response onto the canonical structural shape. */ -function drillNormalizeResponse(observed: unknown): { - role: unknown; - contentNonEmpty: boolean; - finishReasonInEnum: boolean; -} { - const { role, content, finish } = drillExtractChatStructure(observed); - return { - role, - contentNonEmpty: typeof content === "string" && content.trim().length > 0, - finishReasonInEnum: DRILL_FINISH_REASONS.has(finish as string), - }; -} - -/** - * Live plane is enabled iff a base URL AND an API key resolved from env. - * `$env` refs resolve to "" when unset, so this is false in replay-only - * (no-creds) runs and true only when the vector's endpoint+apiKey env vars are - * both populated. - */ -function drillLiveEnabled(input: any): boolean { - return Boolean(input.endpoint && input.apiKey); -} - -/** - * Run the drill: bind transport, replay the cassette, assert 3 planes. Returns - * the raw replay response; the runner then applies `normalize` and compares - * against the vector `expected` (the semantic plane). - */ -async function drillInvoke( - input: any, - context: AdapterContext, -): Promise { - const vector = (context.vector ?? {}) as Record; - const exchangeRaw = (vector.exchange ?? {}) as Record; - // The runner resolves refs only on `input`; the adapter owns `exchange` - // resolution (nested `$env` transport ref + `$json` cassette ref). - const exchange = context.resolveInput(exchangeRaw) as Record; - - const cassette = (exchange.cassette ?? {}) as Record; - const cassetteRequest = (cassette.request ?? {}) as Record; - const cassetteResponse = (cassette.response ?? {}) as Record; - const recordedRequest = cassetteRequest.body; - const recordedResponse = cassetteResponse.body; - const recordedStatus = Number(cassetteResponse.status ?? 200); - if (recordedResponse === undefined) { - throw new Error("drill cassette missing response.body"); - } - - // --- Replay: bind the transport to the local cassette server ------------- - const server = new CassetteReplayServer(recordedResponse, recordedStatus); - let raw: unknown; - await server.start(); - try { - raw = await drillExecute(input, { - endpoint: server.baseUrl, - apiKey: DRILL_REPLAY_API_KEY, - }); - - // Plane 1: transport -- the seam was reached and a request was captured. - const captured = server.lastRequest; - if (captured === null) { - throw new Error( - "drill transport plane: no request reached the replay server", - ); - } - if (raw === null || raw === undefined) { - throw new Error("drill transport plane: executor returned no response"); - } - - // Plane 2: wire -- outbound request body matches the cassette request. - if (recordedRequest !== undefined) { - const observedBody = captured.body; - if (drillCanonical(observedBody) !== drillCanonical(recordedRequest)) { - throw new Error( - "drill wire plane mismatch:\n" + - ` observed=${drillCanonical(observedBody)}\n` + - ` cassette=${drillCanonical(recordedRequest)}`, - ); - } - } - } finally { - await server.stop(); - } - - // --- Live parity (self-gated): live-semantic must equal replay-semantic --- - if (drillLiveEnabled(input)) { - const liveRaw = await drillExecute(input, { - endpoint: input.endpoint || null, - apiKey: input.apiKey, - }); - if ( - drillCanonical(drillNormalizeResponse(liveRaw)) !== - drillCanonical(drillNormalizeResponse(raw)) - ) { - throw new Error( - "drill live/replay parity mismatch:\n" + - ` live=${drillCanonical(drillNormalizeResponse(liveRaw))}\n` + - ` replay=${drillCanonical(drillNormalizeResponse(raw))}`, - ); - } - } - - // Plane 3 (semantic) is asserted by the runner: normalize(raw) == expected. - return raw; -} - -/** Registered semantic projection: raw response -> canonical structure. */ -function drillNormalize(observed: unknown, _context: AdapterContext): unknown { - return drillNormalizeResponse(observed); -} - // --------------------------------------------------------------------------- // Adapter registry // --------------------------------------------------------------------------- @@ -1332,10 +1076,6 @@ export const vectorAdapters = { "TurnConformance.replay": { invoke: replayInvoke }, "DiscoveryConformance.enrich": { invoke: discoveryEnrichInvoke }, "DiscoveryConformance.mapModel": { invoke: discoveryMapInvoke }, - "LiveChatConformance.complete": { - invoke: drillInvoke, - normalize: drillNormalize, - }, }; // Every cross-runtime contract now has a real TypeScript adapter above; there diff --git a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts index 95027d62d..11ba18e58 100644 --- a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts +++ b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts @@ -130,12 +130,6 @@ describe("callable vector conformance", () => { ) as Record; await runVector("DiscoveryConformance", "mapModel", vector, false, seam); }); - it("LiveChatConformance.complete:openai_chat_drill", async () => { - const vector = JSON.parse( - '{"name":"openai_chat_drill","stage":"drill","provider":"openai","input":{"provider":"openai","model":"gpt-4o-mini","apiKey":{"$env":"OPENAI_API_KEY"},"endpoint":{"$env":"OPENAI_BASE_URL"},"messages":[{"role":"user","content":"Say hello in exactly one word."}],"options":{"temperature":0,"maxOutputTokens":16}},"exchange":{"transport":{"baseUrl":{"$env":"OPENAI_BASE_URL"}},"cassette":{"$json":"cassettes/openai_chat_drill.json"},"planes":["transport","wire","semantic"]},"expected":{"role":"assistant","contentNonEmpty":true,"finishReasonInEnum":true},"operation":"complete"}', - ) as Record; - await runVector("LiveChatConformance", "complete", vector, false, seam); - }); it("LoadConformance.load:basic_load", async () => { const vector = JSON.parse( '{"name":"basic_load","description":"Load basic.prompty with env var resolution and verify all PromptAgent fields","stage":"load","input":{"fixture":"basic.prompty","env":{"OPENAI_ENDPOINT":"https://test.openai.com","OPENAI_API_KEY":"sk-test123"}},"expected":{"kind":"prompt","name":"basic-prompt","description":"A basic prompt for testing","metadata":{"authors":["testauthor"]},"model":{"id":"gpt-4","provider":"openai","apiType":"chat","connection":{"kind":"key","endpoint":"https://test.openai.com"},"options":{"temperature":0.7,"maxOutputTokens":1000}},"inputs":[{"name":"firstName","kind":"string","default":"Jane"},{"name":"lastName","kind":"string","default":"Doe"},{"name":"question","kind":"string","default":"What is the meaning of life?"}],"template":{"format":{"kind":"jinja2"},"parser":{"kind":"prompty"}},"instructions":"system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}"},"operation":"load"}', diff --git a/schema/model/conformance/seams/live-chat-conformance.tsp b/schema/model/conformance/seams/live-chat-conformance.tsp deleted file mode 100644 index cb3581e92..000000000 --- a/schema/model/conformance/seams/live-chat-conformance.tsp +++ /dev/null @@ -1,24 +0,0 @@ -import "@typra/emitter"; - -using Prompty.Conformance.Vectors; -namespace Prompty.Conformance.Seams; - -/** - * Conformance-only seam hosting the "service drill" vectors: a conformance - * vector that crosses the REAL Executor->Processor transport seam instead of - * stopping at the SDK edge. A drill runs against a live provider when - * credentials are present and REPLAYS a recorded cassette when they are not; - * the live<->replay parity is the gate. - * - * Like every other conformance seam this interface exists purely to carry - * @vector evidence into the emitted cross-runtime vectors.json; no runtime - * implements it directly. The consumer-owned drill adapter (roster) reads the - * vector's `exchange` block -- an UNKNOWN key that rides through the emitter - * untouched -- to bind the transport and assert the transport/wire/semantic - * planes. Nothing in Typra needs to know the word "drill". - */ -interface LiveChatConformance { - @doc("Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes") - @vector(DrillVectors) - complete(input: unknown): unknown; -} diff --git a/schema/model/conformance/vectors/drill.tsp b/schema/model/conformance/vectors/drill.tsp deleted file mode 100644 index 467804478..000000000 --- a/schema/model/conformance/vectors/drill.tsp +++ /dev/null @@ -1,55 +0,0 @@ -import "@typra/emitter"; - -namespace Prompty.Conformance.Vectors; - -// The first prompty "service drill" -- a conformance vector that crosses the -// REAL Executor->Processor transport seam. Bound to LiveChatConformance.complete. -// -// Authored as a JSON-STRING vector set (Typra 0.7.0+ affordance) so the domain -// request may carry keyword keys (e.g. "model") and nested ref objects. Two keys -// are intentionally NON-STANDARD and prove the emitter's unknown-key passthrough: -// -// * "exchange" -- the drill's transport/cassette/plane contract. The runner -// does NOT resolve it (it only resolves "input"); the consumer-owned adapter -// reads context.vector["exchange"] and resolves the nested $env base-url ref -// and $json cassette ref ITSELF via context.resolveInput. -// -// "requires" is deliberately OMITTED. The runner's requirement guard is -// all-or-nothing (an unavailable token skips the WHOLE vector), which cannot -// express "defer the live plane but still assert replay". So the drill vector -// carries no requirement token; the adapter self-gates the live plane on -// base-url + api-key presence and ALWAYS asserts the replay plane. -const DrillVectors = """ - [ - { - "name": "openai_chat_drill", - "stage": "drill", - "provider": "openai", - "input": { - "provider": "openai", - "model": "gpt-4o-mini", - "apiKey": { "$env": "OPENAI_API_KEY" }, - "endpoint": { "$env": "OPENAI_BASE_URL" }, - "messages": [ - { "role": "user", "content": "Say hello in exactly one word." } - ], - "options": { - "temperature": 0, - "maxOutputTokens": 16 - } - }, - "exchange": { - "transport": { - "baseUrl": { "$env": "OPENAI_BASE_URL" } - }, - "cassette": { "$json": "cassettes/openai_chat_drill.json" }, - "planes": ["transport", "wire", "semantic"] - }, - "expected": { - "role": "assistant", - "contentNonEmpty": true, - "finishReasonInEnum": true - } - } - ] - """; diff --git a/schema/model/main.tsp b/schema/model/main.tsp index 4a865ac23..a74a98768 100644 --- a/schema/model/main.tsp +++ b/schema/model/main.tsp @@ -82,10 +82,6 @@ import "./conformance/seams/load-conformance.tsp"; import "./conformance/seams/wire-conformance.tsp"; import "./conformance/seams/discovery-conformance.tsp"; import "./conformance/seams/turn-conformance.tsp"; -// Service-drill seam: carries the transport-crossing drill vector. Emitter-change -// free by design -- the drill's non-standard "exchange" key rides through as an -// unknown vector key and is consumed only by the runtime's roster adapter. -import "./conformance/seams/live-chat-conformance.tsp"; // Behavior vectors (conformance evidence attached to callable operations) import "./conformance/vectors/render.tsp"; @@ -110,10 +106,6 @@ import "./conformance/vectors/replay.tsp"; // spec/vectors/process/stream_failure_vectors.json into the typra vector system so // every runtime is held to the same streaming-failure semantics. import "./conformance/vectors/stream.tsp"; -// Service-drill vector (LiveChatConformance.complete): the first vector to cross -// the real Executor->Processor transport seam. Carries a non-standard "exchange" -// block that the emitter passes through untouched into every runtime's harness. -import "./conformance/vectors/drill.tsp"; // Events import "./contracts/events/payloads.tsp"; diff --git a/schema/tsp-output/.typra-generated/export-surfaces.json b/schema/tsp-output/.typra-generated/export-surfaces.json index 58515d065..bf06d676b 100644 --- a/schema/tsp-output/.typra-generated/export-surfaces.json +++ b/schema/tsp-output/.typra-generated/export-surfaces.json @@ -106,7 +106,6 @@ "InvocationUsage", "InvokerError", "Jinja2Format", - "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -289,13 +288,6 @@ "source": "HostToolExecutor.cs", "protocol": true }, - { - "name": "LiveChatConformance", - "kind": "type", - "group": "", - "source": "LiveChatConformance.cs", - "protocol": true - }, { "name": "LoadConformance", "kind": "type", @@ -2101,26 +2093,6 @@ } ] }, - { - "name": "LiveChatConformance", - "group": "", - "symbol": "LiveChatConformance", - "source": "LiveChatConformance.cs", - "methods": [ - { - "name": "complete", - "returns": "unknown", - "params": { - "input": "unknown" - }, - "optional": false, - "sync": false, - "runtimeCancellable": false, - "atomic": false, - "nonFatal": false - } - ] - }, { "name": "LoadConformance", "group": "", @@ -2461,7 +2433,6 @@ "EventSink.cs", "Executor.cs", "HostToolExecutor.cs", - "LiveChatConformance.cs", "LoadConformance.cs", "ModelLister.cs", "Parser.cs", @@ -2674,7 +2645,6 @@ "InvocationUsage", "InvokerError", "Jinja2Format", - "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -2857,13 +2827,6 @@ "source": "host_tool_executor.go", "protocol": true }, - { - "name": "LiveChatConformance", - "kind": "type", - "group": "", - "source": "live_chat_conformance.go", - "protocol": true - }, { "name": "LoadConformance", "kind": "type", @@ -4669,26 +4632,6 @@ } ] }, - { - "name": "LiveChatConformance", - "group": "", - "symbol": "LiveChatConformance", - "source": "live_chat_conformance.go", - "methods": [ - { - "name": "complete", - "returns": "unknown", - "params": { - "input": "unknown" - }, - "optional": false, - "sync": false, - "runtimeCancellable": false, - "atomic": false, - "nonFatal": false - } - ] - }, { "name": "LoadConformance", "group": "", @@ -5074,7 +5017,6 @@ "invocation_context_state.go", "invocation_usage.go", "invoker_error.go", - "live_chat_conformance.go", "llm_complete_payload.go", "llm_start_payload.go", "load_conformance.go", @@ -5242,7 +5184,6 @@ "InvocationUsage", "InvokerError", "Jinja2Format", - "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -5425,13 +5366,6 @@ "source": "HostToolExecutor.java", "protocol": true }, - { - "name": "LiveChatConformance", - "kind": "type", - "group": "", - "source": "LiveChatConformance.java", - "protocol": true - }, { "name": "LoadConformance", "kind": "type", @@ -7237,26 +7171,6 @@ } ] }, - { - "name": "LiveChatConformance", - "group": "", - "symbol": "LiveChatConformance", - "source": "LiveChatConformance.java", - "methods": [ - { - "name": "complete", - "returns": "unknown", - "params": { - "input": "unknown" - }, - "optional": false, - "sync": false, - "runtimeCancellable": false, - "atomic": false, - "nonFatal": false - } - ] - }, { "name": "LoadConformance", "group": "", @@ -7642,7 +7556,6 @@ "InvocationContextState.java", "InvocationUsage.java", "InvokerError.java", - "LiveChatConformance.java", "LlmCompletePayload.java", "LlmStartPayload.java", "LoadConformance.java", @@ -7809,7 +7722,6 @@ "InvocationUsage", "InvokerError", "Jinja2Format", - "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -7992,13 +7904,6 @@ "source": "HostToolExecutor", "protocol": true }, - { - "name": "LiveChatConformance", - "kind": "type", - "group": "", - "source": "LiveChatConformance", - "protocol": true - }, { "name": "LoadConformance", "kind": "type", @@ -9804,26 +9709,6 @@ } ] }, - { - "name": "LiveChatConformance", - "group": "", - "symbol": "LiveChatConformance", - "source": "LiveChatConformance", - "methods": [ - { - "name": "complete", - "returns": "unknown", - "params": { - "input": "unknown" - }, - "optional": false, - "sync": false, - "runtimeCancellable": false, - "atomic": false, - "nonFatal": false - } - ] - }, { "name": "LoadConformance", "group": "", @@ -10209,7 +10094,6 @@ "InvocationContextState", "InvocationUsage", "InvokerError", - "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -10377,7 +10261,6 @@ "InvocationUsage", "InvokerError", "Jinja2Format", - "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -10560,13 +10443,6 @@ "source": "._HostToolExecutor", "protocol": true }, - { - "name": "LiveChatConformance", - "kind": "type", - "group": "", - "source": "._LiveChatConformance", - "protocol": true - }, { "name": "LoadConformance", "kind": "type", @@ -12372,26 +12248,6 @@ } ] }, - { - "name": "LiveChatConformance", - "group": "", - "symbol": "LiveChatConformance", - "source": "._LiveChatConformance", - "methods": [ - { - "name": "complete", - "returns": "unknown", - "params": { - "input": "unknown" - }, - "optional": false, - "sync": false, - "runtimeCancellable": false, - "atomic": false, - "nonFatal": false - } - ] - }, { "name": "LoadConformance", "group": "", @@ -12732,7 +12588,6 @@ "._EventSink", "._Executor", "._HostToolExecutor", - "._LiveChatConformance", "._LoadConformance", "._ModelLister", "._Parser", @@ -12822,7 +12677,6 @@ "InvocationUsage", "InvokerError", "Jinja2Format", - "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -13005,13 +12859,6 @@ "source": "host_tool_executor", "protocol": true }, - { - "name": "LiveChatConformance", - "kind": "type", - "group": "", - "source": "live_chat_conformance", - "protocol": true - }, { "name": "LoadConformance", "kind": "type", @@ -14817,26 +14664,6 @@ } ] }, - { - "name": "LiveChatConformance", - "group": "", - "symbol": "LiveChatConformance", - "source": "live_chat_conformance", - "methods": [ - { - "name": "complete", - "returns": "unknown", - "params": { - "input": "unknown" - }, - "optional": false, - "sync": false, - "runtimeCancellable": false, - "atomic": false, - "nonFatal": false - } - ] - }, { "name": "LoadConformance", "group": "", @@ -15179,7 +15006,6 @@ "event_sink", "executor", "host_tool_executor", - "live_chat_conformance", "load_conformance", "model_lister", "operations", @@ -15269,7 +15095,6 @@ "InvocationUsage", "InvokerError", "Jinja2Format", - "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -15452,13 +15277,6 @@ "source": "Sources/PromptyModel/host_tool_executor.swift", "protocol": true }, - { - "name": "LiveChatConformance", - "kind": "type", - "group": "", - "source": "Sources/PromptyModel/live_chat_conformance.swift", - "protocol": true - }, { "name": "LoadConformance", "kind": "type", @@ -17264,26 +17082,6 @@ } ] }, - { - "name": "LiveChatConformance", - "group": "", - "symbol": "LiveChatConformance", - "source": "Sources/PromptyModel/live_chat_conformance.swift", - "methods": [ - { - "name": "complete", - "returns": "unknown", - "params": { - "input": "unknown" - }, - "optional": false, - "sync": false, - "runtimeCancellable": false, - "atomic": false, - "nonFatal": false - } - ] - }, { "name": "LoadConformance", "group": "", @@ -17738,7 +17536,6 @@ "Sources/PromptyModel/event_sink.swift", "Sources/PromptyModel/executor.swift", "Sources/PromptyModel/host_tool_executor.swift", - "Sources/PromptyModel/live_chat_conformance.swift", "Sources/PromptyModel/load_conformance.swift", "Sources/PromptyModel/model_lister.swift", "Sources/PromptyModel/operations/render_segment.swift", @@ -17837,7 +17634,6 @@ "InvocationUsage", "InvokerError", "Jinja2Format", - "LiveChatConformance", "LlmCompletePayload", "LlmStartPayload", "LoadConformance", @@ -18020,13 +17816,6 @@ "source": "./host-tool-executor", "protocol": true }, - { - "name": "LiveChatConformance", - "kind": "type", - "group": "", - "source": "./live-chat-conformance", - "protocol": true - }, { "name": "LoadConformance", "kind": "type", @@ -19832,26 +19621,6 @@ } ] }, - { - "name": "LiveChatConformance", - "group": "", - "symbol": "LiveChatConformance", - "source": "./live-chat-conformance", - "methods": [ - { - "name": "complete", - "returns": "unknown", - "params": { - "input": "unknown" - }, - "optional": false, - "sync": false, - "runtimeCancellable": false, - "atomic": false, - "nonFatal": false - } - ] - }, { "name": "LoadConformance", "group": "", @@ -20306,7 +20075,6 @@ "./event-sink", "./executor", "./host-tool-executor", - "./live-chat-conformance", "./load-conformance", "./model-lister", "./operations/render-segment", diff --git a/schema/tsp-output/.typra-generated/hydration-seams.json b/schema/tsp-output/.typra-generated/hydration-seams.json index 3075ffb6b..b9d874328 100644 --- a/schema/tsp-output/.typra-generated/hydration-seams.json +++ b/schema/tsp-output/.typra-generated/hydration-seams.json @@ -84,14 +84,6 @@ "generatedSource": "HostToolExecutor.cs", "seamKind": "protocol-adapter" }, - { - "contract": "LiveChatConformance", - "target": "csharp", - "group": "", - "symbol": "LiveChatConformance", - "generatedSource": "LiveChatConformance.cs", - "seamKind": "protocol-adapter" - }, { "contract": "LoadConformance", "target": "csharp", @@ -236,14 +228,6 @@ "generatedSource": "host_tool_executor.go", "seamKind": "protocol-adapter" }, - { - "contract": "LiveChatConformance", - "target": "go", - "group": "", - "symbol": "LiveChatConformance", - "generatedSource": "live_chat_conformance.go", - "seamKind": "protocol-adapter" - }, { "contract": "LoadConformance", "target": "go", @@ -388,14 +372,6 @@ "generatedSource": "HostToolExecutor.java", "seamKind": "protocol-adapter" }, - { - "contract": "LiveChatConformance", - "target": "java", - "group": "", - "symbol": "LiveChatConformance", - "generatedSource": "LiveChatConformance.java", - "seamKind": "protocol-adapter" - }, { "contract": "LoadConformance", "target": "java", @@ -540,14 +516,6 @@ "generatedSource": "HostToolExecutor", "seamKind": "protocol-adapter" }, - { - "contract": "LiveChatConformance", - "target": "markdown", - "group": "", - "symbol": "LiveChatConformance", - "generatedSource": "LiveChatConformance", - "seamKind": "protocol-adapter" - }, { "contract": "LoadConformance", "target": "markdown", @@ -692,14 +660,6 @@ "generatedSource": "._HostToolExecutor", "seamKind": "protocol-adapter" }, - { - "contract": "LiveChatConformance", - "target": "python", - "group": "", - "symbol": "LiveChatConformance", - "generatedSource": "._LiveChatConformance", - "seamKind": "protocol-adapter" - }, { "contract": "LoadConformance", "target": "python", @@ -844,14 +804,6 @@ "generatedSource": "host_tool_executor", "seamKind": "protocol-adapter" }, - { - "contract": "LiveChatConformance", - "target": "rust", - "group": "", - "symbol": "LiveChatConformance", - "generatedSource": "live_chat_conformance", - "seamKind": "protocol-adapter" - }, { "contract": "LoadConformance", "target": "rust", @@ -996,14 +948,6 @@ "generatedSource": "Sources/PromptyModel/host_tool_executor.swift", "seamKind": "protocol-adapter" }, - { - "contract": "LiveChatConformance", - "target": "swift", - "group": "", - "symbol": "LiveChatConformance", - "generatedSource": "Sources/PromptyModel/live_chat_conformance.swift", - "seamKind": "protocol-adapter" - }, { "contract": "LoadConformance", "target": "swift", @@ -1148,14 +1092,6 @@ "generatedSource": "./host-tool-executor", "seamKind": "protocol-adapter" }, - { - "contract": "LiveChatConformance", - "target": "typescript", - "group": "", - "symbol": "LiveChatConformance", - "generatedSource": "./live-chat-conformance", - "seamKind": "protocol-adapter" - }, { "contract": "LoadConformance", "target": "typescript", diff --git a/schema/tsp-output/.typra-generated/manifest.json b/schema/tsp-output/.typra-generated/manifest.json index f58c5dbb1..ff712eb29 100644 --- a/schema/tsp-output/.typra-generated/manifest.json +++ b/schema/tsp-output/.typra-generated/manifest.json @@ -1333,11 +1333,6 @@ "path": "../runtime/csharp/Prompty.Core/Model/HostToolExecutor.cs", "marker": true }, - { - "outputRoot": "../runtime/csharp/Prompty.Core/Model", - "path": "../runtime/csharp/Prompty.Core/Model/LiveChatConformance.cs", - "marker": true - }, { "outputRoot": "../runtime/csharp/Prompty.Core/Model", "path": "../runtime/csharp/Prompty.Core/Model/LoadConformance.cs", @@ -1908,11 +1903,6 @@ "path": "../runtime/go/prompty/model/jinja2format_test.go", "marker": true }, - { - "outputRoot": "../runtime/go/prompty/model", - "path": "../runtime/go/prompty/model/live_chat_conformance.go", - "marker": true - }, { "outputRoot": "../runtime/go/prompty/model", "path": "../runtime/go/prompty/model/llm_complete_payload.go", @@ -3043,11 +3033,6 @@ "path": "../runtime/java/prompty/src/main/java/com/microsoft/prompty/model/Jinja2Format.java", "marker": true }, - { - "outputRoot": "../runtime/java/prompty/src/main/java/com/microsoft/prompty/model", - "path": "../runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LiveChatConformance.java", - "marker": true - }, { "outputRoot": "../runtime/java/prompty/src/main/java/com/microsoft/prompty/model", "path": "../runtime/java/prompty/src/main/java/com/microsoft/prompty/model/LlmCompletePayload.java", @@ -4213,11 +4198,6 @@ "path": "../runtime/python/prompty/prompty/model/_HostToolExecutor.py", "marker": true }, - { - "outputRoot": "../runtime/python/prompty/prompty/model", - "path": "../runtime/python/prompty/prompty/model/_LiveChatConformance.py", - "marker": true - }, { "outputRoot": "../runtime/python/prompty/prompty/model", "path": "../runtime/python/prompty/prompty/model/_LoadConformance.py", @@ -6123,11 +6103,6 @@ "path": "../runtime/rust/prompty/src/model/host_tool_executor.rs", "marker": true }, - { - "outputRoot": "../runtime/rust/prompty/src/model", - "path": "../runtime/rust/prompty/src/model/live_chat_conformance.rs", - "marker": true - }, { "outputRoot": "../runtime/rust/prompty/src/model", "path": "../runtime/rust/prompty/src/model/load_conformance.rs", @@ -7238,11 +7213,6 @@ "path": "../runtime/swift/prompty-model/Sources/PromptyModel/host_tool_executor.swift", "marker": true }, - { - "outputRoot": "../runtime/swift/prompty-model", - "path": "../runtime/swift/prompty-model/Sources/PromptyModel/live_chat_conformance.swift", - "marker": true - }, { "outputRoot": "../runtime/swift/prompty-model", "path": "../runtime/swift/prompty-model/Sources/PromptyModel/load_conformance.swift", @@ -8343,11 +8313,6 @@ "path": "../runtime/typescript/packages/core/src/model/index.ts", "marker": true }, - { - "outputRoot": "../runtime/typescript/packages/core/src/model", - "path": "../runtime/typescript/packages/core/src/model/live-chat-conformance.ts", - "marker": true - }, { "outputRoot": "../runtime/typescript/packages/core/src/model", "path": "../runtime/typescript/packages/core/src/model/load-conformance.ts", @@ -9268,11 +9233,6 @@ "path": "../web/src/content/docs/reference/Jinja2Format.md", "marker": true }, - { - "outputRoot": "../web/src/content/docs/reference", - "path": "../web/src/content/docs/reference/LiveChatConformance.md", - "marker": true - }, { "outputRoot": "../web/src/content/docs/reference", "path": "../web/src/content/docs/reference/LlmCompletePayload.md", diff --git a/schema/tsp-output/.typra-generated/vectors.json b/schema/tsp-output/.typra-generated/vectors.json index 7da1d9af3..f9a332c06 100644 --- a/schema/tsp-output/.typra-generated/vectors.json +++ b/schema/tsp-output/.typra-generated/vectors.json @@ -567,63 +567,6 @@ "operation": "mapModel" } }, - { - "contract": "LiveChatConformance", - "namespace": "Prompty.Conformance.Seams", - "group": "", - "operation": "complete", - "params": { - "input": "unknown" - }, - "returns": "unknown", - "sync": false, - "vector": { - "name": "openai_chat_drill", - "stage": "drill", - "provider": "openai", - "input": { - "provider": "openai", - "model": "gpt-4o-mini", - "apiKey": { - "$env": "OPENAI_API_KEY" - }, - "endpoint": { - "$env": "OPENAI_BASE_URL" - }, - "messages": [ - { - "role": "user", - "content": "Say hello in exactly one word." - } - ], - "options": { - "temperature": 0, - "maxOutputTokens": 16 - } - }, - "exchange": { - "transport": { - "baseUrl": { - "$env": "OPENAI_BASE_URL" - } - }, - "cassette": { - "$json": "cassettes/openai_chat_drill.json" - }, - "planes": [ - "transport", - "wire", - "semantic" - ] - }, - "expected": { - "role": "assistant", - "contentNonEmpty": true, - "finishReasonInEnum": true - }, - "operation": "complete" - } - }, { "contract": "LoadConformance", "namespace": "Prompty.Conformance.Seams", diff --git a/web/src/content/docs/reference/LiveChatConformance.md b/web/src/content/docs/reference/LiveChatConformance.md deleted file mode 100644 index e5a54d3df..000000000 --- a/web/src/content/docs/reference/LiveChatConformance.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: "LiveChatConformance" -description: "Documentation for the LiveChatConformance type." -slug: "reference/livechatconformance" ---- - - -Conformance-only seam hosting the "service drill" vectors: a conformance -vector that crosses the REAL Executor->Processor transport seam instead of -stopping at the SDK edge. A drill runs against a live provider when -credentials are present and REPLAYS a recorded cassette when they are not; -the live<->replay parity is the gate. - -Like every other conformance seam this interface exists purely to carry - -## Class Diagram - -```mermaid ---- -title: LiveChatConformance -config: - look: handDrawn - theme: colorful - class: - hideEmptyMembersBox: true ---- -classDiagram - class LiveChatConformance { - <> - +complete(input: unknown) unknown [async-capable] - } -``` - -## Helper Methods - -The following helper methods are declared via `@method` and must be implemented by every runtime. The schema declares the logical protocol contract; each runtime maps async-capable methods to idiomatic sync/async shapes for that language. - -| Name | Signature | Runtime shape | Description | -| ---- | --------- | ------------- | ----------- | -| `complete` | `complete(input: unknown) -> unknown` | async-capable | Drive a real chat completion across the transport seam (live or cassette-replay) and assert 3 planes |