From bdc3ad2ef0f912ddbe94a67e9c48b073a39c06e2 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 12:12:15 -0700 Subject: [PATCH 01/15] feat(spec): require trusted endpoints for ambient credentials Add spec section 2.5.1 Endpoint Trust, the untrusted_endpoint load error, the trustedEndpoints load option, and load conformance vectors. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Model/VectorConformanceTests.cs | 672 +++++++++++++++--- .../Prompty.Core/Model/VectorConformance.cs | 168 +++++ .../prompty/model/vector_conformance_test.go | 657 ++++++++++++++--- .../vectorconformance/vector_conformance.go | 255 +++++++ .../prompty/model/VectorConformance.java | 208 +++++- .../prompty/model/VectorConformanceTests.java | 444 +++++++----- .../prompty/model/vector_conformance.py | 148 ++++ .../tests/model/test_vector_conformance.py | 642 ++++++++++++++--- .../prompty/src/model/vector_conformance.rs | 312 ++++++++ .../tests/model/vector_conformance_test.rs | 269 ++++--- .../PromptyModel/vector_conformance.swift | 241 ++++++- .../VectorConformanceTests.swift | 314 +++++--- .../core/src/model/vector-conformance.ts | 219 ++++++ .../tests/model/vector-conformance.test.ts | 94 ++- schema/model/conformance/vectors/load.tsp | 356 +++++++++- .../tsp-output/.typra-generated/vectors.json | 542 +++++++++++++- spec/spec.md | 69 +- 17 files changed, 5023 insertions(+), 587 deletions(-) diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs index 0a7740e7c..acc0f91c5 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs @@ -828,7 +828,10 @@ public async Task Vector21LoadConformanceLoadConnectionTypesLoad() "endpoint": "https://localhost:8080" } } - } + }, + "trustedEndpoints": [ + "https://localhost:8080" + ] }, "expected": { "model": { @@ -931,7 +934,8 @@ public async Task Vector24LoadConformanceLoadEnvDefault() "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "${env:MISSING_VAR:fallback_value}" + "endpoint": "${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }, @@ -1793,7 +1797,511 @@ public async Task Vector47LoadConformanceLoadToolsOpenapiLoad() } [Fact] - public async Task Vector48MemoryConformanceOperateClearByCategoryAndAll() + public async Task Vector48LoadConformanceLoadTrustedEndpointBuiltinAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_builtin_allowed", + "description": "Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-builtin", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_env_endpoint_allowed", + "description": "An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-endpoint", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:VECTOR_FOUNDRY_ENDPOINT}" + } + } + }, + "env": { + "VECTOR_FOUNDRY_ENDPOINT": "https://host-chosen.example.com" + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://host-chosen.example.com" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector50LoadConformanceLoadTrustedEndpointEnvListAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_env_list_allowed", + "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-list", + "model": { + "id": "llama3", + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "env": { + "PROMPTY_TRUSTED_ENDPOINTS": "https://other.example.com, http://localhost:11434" + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_literal_key_allowed", + "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-literal-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com", + "apiKey": "file-supplied-key" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector52LoadConformanceLoadTrustedEndpointOptionAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_option_allowed", + "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-option", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "trustedEndpoints": [ + "HTTPS://PROXY.Example.com" + ] + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector53LoadConformanceLoadTrustedEndpointReferenceSkipped() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_reference_skipped", + "description": "kind: reference connections carry no endpoint and are not subject to the trust check", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-reference", + "model": { + "id": "gpt-4", + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector54LoadConformanceLoadUntrustedEndpointAmbientRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_ambient_rejected", + "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec \u00a72.5.1)", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ambient", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com/api/projects/p" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_env_default_rejected", + "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-default", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:NONEXISTENT:https://attacker.example.com}" + } + } + }, + "env": {} + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_env_key_rejected", + "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": "${env:VECTOR_HOST_KEY}" + } + } + }, + "env": { + "VECTOR_HOST_KEY": "host-secret" + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_file_connection_rejected", + "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-file-conn", + "model": { + "id": "gpt-4", + "connection": "${file:conn.json}" + } + }, + "files": { + "conn.json": { + "kind": "foundry", + "endpoint": "https://attacker.example.com" + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_key_with_client_secret_rejected", + "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-key-client-secret", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "clientSecret": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_mismatched_credential_rejected", + "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-mismatched-credential", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com", + "apiKey": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector60LoadConformanceLoadUntrustedEndpointNoKeyRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_no_key_rejected", + "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-no-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "http://127.0.0.1:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_suffix_spoof_rejected", + "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-spoof", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://good.example.com.attacker.example.com" + } + } + }, + "trustedEndpoints": [ + "https://good.example.com" + ] + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_wildcard_apex_rejected", + "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-apex", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://myres.openai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector63MemoryConformanceOperateClearByCategoryAndAll() { string vectorJson = """ { @@ -1847,7 +2355,7 @@ public async Task Vector48MemoryConformanceOperateClearByCategoryAndAll() } [Fact] - public async Task Vector49MemoryConformanceOperateFormatCoreAndRecallResults() + public async Task Vector64MemoryConformanceOperateFormatCoreAndRecallResults() { string vectorJson = """ { @@ -1889,7 +2397,7 @@ public async Task Vector49MemoryConformanceOperateFormatCoreAndRecallResults() } [Fact] - public async Task Vector50MemoryConformanceOperateRecallEmptyQueryAndLimit() + public async Task Vector65MemoryConformanceOperateRecallEmptyQueryAndLimit() { string vectorJson = """ { @@ -1944,7 +2452,7 @@ public async Task Vector50MemoryConformanceOperateRecallEmptyQueryAndLimit() } [Fact] - public async Task Vector51MemoryConformanceOperateRecallRanksByWeightedScore() + public async Task Vector66MemoryConformanceOperateRecallRanksByWeightedScore() { string vectorJson = """ { @@ -2003,7 +2511,7 @@ public async Task Vector51MemoryConformanceOperateRecallRanksByWeightedScore() } [Fact] - public async Task Vector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() + public async Task Vector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() { string vectorJson = """ { @@ -2061,7 +2569,7 @@ public async Task Vector52MemoryConformanceOperateRecallScoreTiesPreserveInserti } [Fact] - public async Task Vector53MemoryConformanceOperateRecallTagsAndCoreBoost() + public async Task Vector68MemoryConformanceOperateRecallTagsAndCoreBoost() { string vectorJson = """ { @@ -2122,7 +2630,7 @@ public async Task Vector53MemoryConformanceOperateRecallTagsAndCoreBoost() } [Fact] - public async Task Vector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() + public async Task Vector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() { string vectorJson = """ { @@ -2183,7 +2691,7 @@ public async Task Vector54MemoryConformanceOperateRecallTokenizationPunctuationU } [Fact] - public async Task Vector55MemoryConformanceOperateRememberDedupsCoreByTags() + public async Task Vector70MemoryConformanceOperateRememberDedupsCoreByTags() { string vectorJson = """ { @@ -2247,7 +2755,7 @@ public async Task Vector55MemoryConformanceOperateRememberDedupsCoreByTags() } [Fact] - public async Task Vector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() + public async Task Vector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() { string vectorJson = """ { @@ -2303,7 +2811,7 @@ public async Task Vector56MemoryConformanceOperateRememberDedupsCoreMissingAndEm } [Fact] - public async Task Vector57MemoryConformanceOperateRememberEvictsArchivalFirst() + public async Task Vector72MemoryConformanceOperateRememberEvictsArchivalFirst() { string vectorJson = """ { @@ -2364,7 +2872,7 @@ public async Task Vector57MemoryConformanceOperateRememberEvictsArchivalFirst() } [Fact] - public async Task Vector58MemoryConformanceOperateSnapshotPortRoundTrip() + public async Task Vector73MemoryConformanceOperateSnapshotPortRoundTrip() { string vectorJson = """ { @@ -2404,7 +2912,7 @@ public async Task Vector58MemoryConformanceOperateSnapshotPortRoundTrip() } [Fact] - public async Task Vector59MemoryConformanceOperateUpdateIndexOutOfRangeError() + public async Task Vector74MemoryConformanceOperateUpdateIndexOutOfRangeError() { string vectorJson = """ { @@ -2438,7 +2946,7 @@ public async Task Vector59MemoryConformanceOperateUpdateIndexOutOfRangeError() } [Fact] - public async Task Vector60TurnConformanceReplayMaxIterations() + public async Task Vector75TurnConformanceReplayMaxIterations() { string vectorJson = """ { @@ -2475,7 +2983,7 @@ public async Task Vector60TurnConformanceReplayMaxIterations() } [Fact] - public async Task Vector61TurnConformanceReplayNoTool() + public async Task Vector76TurnConformanceReplayNoTool() { string vectorJson = """ { @@ -2508,7 +3016,7 @@ public async Task Vector61TurnConformanceReplayNoTool() } [Fact] - public async Task Vector62TurnConformanceReplayPermissionDenied() + public async Task Vector77TurnConformanceReplayPermissionDenied() { string vectorJson = """ { @@ -2544,7 +3052,7 @@ public async Task Vector62TurnConformanceReplayPermissionDenied() } [Fact] - public async Task Vector63TurnConformanceReplayToolFailure() + public async Task Vector78TurnConformanceReplayToolFailure() { string vectorJson = """ { @@ -2582,7 +3090,7 @@ public async Task Vector63TurnConformanceReplayToolFailure() } [Fact] - public async Task Vector64TurnConformanceReplayToolSuccess() + public async Task Vector79TurnConformanceReplayToolSuccess() { string vectorJson = """ { @@ -2620,7 +3128,7 @@ public async Task Vector64TurnConformanceReplayToolSuccess() } [Fact] - public async Task Vector65TurnConformanceRunAssistantToolCallsMetadata() + public async Task Vector80TurnConformanceRunAssistantToolCallsMetadata() { string vectorJson = """ { @@ -2774,7 +3282,7 @@ public async Task Vector65TurnConformanceRunAssistantToolCallsMetadata() } [Fact] - public async Task Vector66TurnConformanceRunAsyncToolFunction() + public async Task Vector81TurnConformanceRunAsyncToolFunction() { string vectorJson = """ { @@ -2886,7 +3394,7 @@ public async Task Vector66TurnConformanceRunAsyncToolFunction() } [Fact] - public async Task Vector67TurnConformanceRunBindingsInjected() + public async Task Vector82TurnConformanceRunBindingsInjected() { string vectorJson = """ { @@ -3018,7 +3526,7 @@ public async Task Vector67TurnConformanceRunBindingsInjected() } [Fact] - public async Task Vector68TurnConformanceRunCancellationBeforeLlm() + public async Task Vector83TurnConformanceRunCancellationBeforeLlm() { string vectorJson = """ { @@ -3081,7 +3589,7 @@ public async Task Vector68TurnConformanceRunCancellationBeforeLlm() } [Fact] - public async Task Vector69TurnConformanceRunCancellationBetweenIterations() + public async Task Vector84TurnConformanceRunCancellationBetweenIterations() { string vectorJson = """ { @@ -3213,7 +3721,7 @@ public async Task Vector69TurnConformanceRunCancellationBetweenIterations() } [Fact] - public async Task Vector70TurnConformanceRunCancellationBetweenTools() + public async Task Vector85TurnConformanceRunCancellationBetweenTools() { string vectorJson = """ { @@ -3354,7 +3862,7 @@ public async Task Vector70TurnConformanceRunCancellationBetweenTools() } [Fact] - public async Task Vector71TurnConformanceRunContextNoTrimWhenFits() + public async Task Vector86TurnConformanceRunContextNoTrimWhenFits() { string vectorJson = """ { @@ -3427,7 +3935,7 @@ public async Task Vector71TurnConformanceRunContextNoTrimWhenFits() } [Fact] - public async Task Vector72TurnConformanceRunContextPreservesSystemMessages() + public async Task Vector87TurnConformanceRunContextPreservesSystemMessages() { string vectorJson = """ { @@ -3536,7 +4044,7 @@ public async Task Vector72TurnConformanceRunContextPreservesSystemMessages() } [Fact] - public async Task Vector73TurnConformanceRunContextTrimBasic() + public async Task Vector88TurnConformanceRunContextTrimBasic() { string vectorJson = """ { @@ -3696,7 +4204,7 @@ public async Task Vector73TurnConformanceRunContextTrimBasic() } [Fact] - public async Task Vector74TurnConformanceRunEmptyToolResult() + public async Task Vector89TurnConformanceRunEmptyToolResult() { string vectorJson = """ { @@ -3813,7 +4321,7 @@ public async Task Vector74TurnConformanceRunEmptyToolResult() } [Fact] - public async Task Vector75TurnConformanceRunEventsBasicToolLoop() + public async Task Vector90TurnConformanceRunEventsBasicToolLoop() { string vectorJson = """ { @@ -3961,7 +4469,7 @@ public async Task Vector75TurnConformanceRunEventsBasicToolLoop() } [Fact] - public async Task Vector76TurnConformanceRunEventsErrorLogged() + public async Task Vector91TurnConformanceRunEventsErrorLogged() { string vectorJson = """ { @@ -4089,7 +4597,7 @@ public async Task Vector76TurnConformanceRunEventsErrorLogged() } [Fact] - public async Task Vector77TurnConformanceRunEventsNoTools() + public async Task Vector92TurnConformanceRunEventsNoTools() { string vectorJson = """ { @@ -4169,7 +4677,7 @@ public async Task Vector77TurnConformanceRunEventsNoTools() } [Fact] - public async Task Vector78TurnConformanceRunGuardrailAllPass() + public async Task Vector93TurnConformanceRunGuardrailAllPass() { string vectorJson = """ { @@ -4294,7 +4802,7 @@ public async Task Vector78TurnConformanceRunGuardrailAllPass() } [Fact] - public async Task Vector79TurnConformanceRunGuardrailInputDeny() + public async Task Vector94TurnConformanceRunGuardrailInputDeny() { string vectorJson = """ { @@ -4352,7 +4860,7 @@ public async Task Vector79TurnConformanceRunGuardrailInputDeny() } [Fact] - public async Task Vector80TurnConformanceRunGuardrailOutputDeny() + public async Task Vector95TurnConformanceRunGuardrailOutputDeny() { string vectorJson = """ { @@ -4432,7 +4940,7 @@ public async Task Vector80TurnConformanceRunGuardrailOutputDeny() } [Fact] - public async Task Vector81TurnConformanceRunGuardrailToolDeny() + public async Task Vector96TurnConformanceRunGuardrailToolDeny() { string vectorJson = """ { @@ -4588,7 +5096,7 @@ public async Task Vector81TurnConformanceRunGuardrailToolDeny() } [Fact] - public async Task Vector82TurnConformanceRunMaxIterationsExceeded() + public async Task Vector97TurnConformanceRunMaxIterationsExceeded() { string vectorJson = """ { @@ -5100,7 +5608,7 @@ public async Task Vector82TurnConformanceRunMaxIterationsExceeded() } [Fact] - public async Task Vector83TurnConformanceRunMultiTurnToolCalls() + public async Task Vector98TurnConformanceRunMultiTurnToolCalls() { string vectorJson = """ { @@ -5279,7 +5787,7 @@ public async Task Vector83TurnConformanceRunMultiTurnToolCalls() } [Fact] - public async Task Vector84TurnConformanceRunMultipleToolCallsSingleTurn() + public async Task Vector99TurnConformanceRunMultipleToolCallsSingleTurn() { string vectorJson = """ { @@ -5462,7 +5970,7 @@ public async Task Vector84TurnConformanceRunMultipleToolCallsSingleTurn() } [Fact] - public async Task Vector85TurnConformanceRunNoToolCalls() + public async Task Vector100TurnConformanceRunNoToolCalls() { string vectorJson = """ { @@ -5532,7 +6040,7 @@ public async Task Vector85TurnConformanceRunNoToolCalls() } [Fact] - public async Task Vector86TurnConformanceRunParallelToolsBasic() + public async Task Vector101TurnConformanceRunParallelToolsBasic() { string vectorJson = """ { @@ -5707,7 +6215,7 @@ public async Task Vector86TurnConformanceRunParallelToolsBasic() } [Fact] - public async Task Vector87TurnConformanceRunParallelToolsWithGuardrailDeny() + public async Task Vector102TurnConformanceRunParallelToolsWithGuardrailDeny() { string vectorJson = """ { @@ -5900,7 +6408,7 @@ public async Task Vector87TurnConformanceRunParallelToolsWithGuardrailDeny() } [Fact] - public async Task Vector88TurnConformanceRunParallelToolsWithToolError() + public async Task Vector103TurnConformanceRunParallelToolsWithToolError() { string vectorJson = """ { @@ -6075,7 +6583,7 @@ public async Task Vector88TurnConformanceRunParallelToolsWithToolError() } [Fact] - public async Task Vector89TurnConformanceRunSingleToolCall() + public async Task Vector104TurnConformanceRunSingleToolCall() { string vectorJson = """ { @@ -6224,7 +6732,7 @@ public async Task Vector89TurnConformanceRunSingleToolCall() } [Fact] - public async Task Vector90TurnConformanceRunSteeringInjectMessage() + public async Task Vector105TurnConformanceRunSteeringInjectMessage() { string vectorJson = """ { @@ -6434,7 +6942,7 @@ public async Task Vector90TurnConformanceRunSteeringInjectMessage() } [Fact] - public async Task Vector91TurnConformanceRunSteeringMultipleMessages() + public async Task Vector106TurnConformanceRunSteeringMultipleMessages() { string vectorJson = """ { @@ -6606,7 +7114,7 @@ public async Task Vector91TurnConformanceRunSteeringMultipleMessages() } [Fact] - public async Task Vector92TurnConformanceRunToolNotRegisteredError() + public async Task Vector107TurnConformanceRunToolNotRegisteredError() { string vectorJson = """ { @@ -6692,7 +7200,7 @@ public async Task Vector92TurnConformanceRunToolNotRegisteredError() } [Fact] - public async Task Vector93TurnConformanceRunToolResultMessageFormat() + public async Task Vector108TurnConformanceRunToolResultMessageFormat() { string vectorJson = """ { @@ -6815,7 +7323,7 @@ public async Task Vector93TurnConformanceRunToolResultMessageFormat() } [Fact] - public async Task Vector94TurnConformanceRunTurnCancelBeforeContext() + public async Task Vector109TurnConformanceRunTurnCancelBeforeContext() { string vectorJson = """ { @@ -6849,7 +7357,7 @@ public async Task Vector94TurnConformanceRunTurnCancelBeforeContext() } [Fact] - public async Task Vector95TurnConformanceRunTurnDelegatedProviderState() + public async Task Vector110TurnConformanceRunTurnDelegatedProviderState() { string vectorJson = """ { @@ -6918,7 +7426,7 @@ public async Task Vector95TurnConformanceRunTurnDelegatedProviderState() } [Fact] - public async Task Vector96TurnConformanceRunTurnFinalOutput() + public async Task Vector111TurnConformanceRunTurnFinalOutput() { string vectorJson = """ { @@ -6965,7 +7473,7 @@ public async Task Vector96TurnConformanceRunTurnFinalOutput() } [Fact] - public async Task Vector97TurnConformanceRunTurnMemoryContextIsModelVisible() + public async Task Vector112TurnConformanceRunTurnMemoryContextIsModelVisible() { string vectorJson = """ { @@ -7029,7 +7537,7 @@ public async Task Vector97TurnConformanceRunTurnMemoryContextIsModelVisible() } [Fact] - public async Task Vector98TurnConformanceRunTurnOrderedToolRound() + public async Task Vector113TurnConformanceRunTurnOrderedToolRound() { string vectorJson = """ { @@ -7122,7 +7630,7 @@ public async Task Vector98TurnConformanceRunTurnOrderedToolRound() } [Fact] - public async Task Vector99TurnConformanceRunTurnPermissionDenialIsModelVisible() + public async Task Vector114TurnConformanceRunTurnPermissionDenialIsModelVisible() { string vectorJson = """ { @@ -7175,7 +7683,7 @@ public async Task Vector99TurnConformanceRunTurnPermissionDenialIsModelVisible() } [Fact] - public async Task Vector100WireConformanceToRequestAnthropicImageFormat() + public async Task Vector115WireConformanceToRequestAnthropicImageFormat() { string vectorJson = """ { @@ -7247,7 +7755,7 @@ public async Task Vector100WireConformanceToRequestAnthropicImageFormat() } [Fact] - public async Task Vector101WireConformanceToRequestAnthropicMaxTokensRequired() + public async Task Vector116WireConformanceToRequestAnthropicMaxTokensRequired() { string vectorJson = """ { @@ -7304,7 +7812,7 @@ public async Task Vector101WireConformanceToRequestAnthropicMaxTokensRequired() } [Fact] - public async Task Vector102WireConformanceToRequestAnthropicOptions() + public async Task Vector117WireConformanceToRequestAnthropicOptions() { string vectorJson = """ { @@ -7375,7 +7883,7 @@ public async Task Vector102WireConformanceToRequestAnthropicOptions() } [Fact] - public async Task Vector103WireConformanceToRequestAnthropicSystemSeparate() + public async Task Vector118WireConformanceToRequestAnthropicSystemSeparate() { string vectorJson = """ { @@ -7444,7 +7952,7 @@ public async Task Vector103WireConformanceToRequestAnthropicSystemSeparate() } [Fact] - public async Task Vector104WireConformanceToRequestAnthropicToolWire() + public async Task Vector119WireConformanceToRequestAnthropicToolWire() { string vectorJson = """ { @@ -7531,7 +8039,7 @@ public async Task Vector104WireConformanceToRequestAnthropicToolWire() } [Fact] - public async Task Vector105WireConformanceToRequestChatAudioMp3() + public async Task Vector120WireConformanceToRequestChatAudioMp3() { string vectorJson = """ { @@ -7591,7 +8099,7 @@ public async Task Vector105WireConformanceToRequestChatAudioMp3() } [Fact] - public async Task Vector106WireConformanceToRequestChatAudioPart() + public async Task Vector121WireConformanceToRequestChatAudioPart() { string vectorJson = """ { @@ -7651,7 +8159,7 @@ public async Task Vector106WireConformanceToRequestChatAudioPart() } [Fact] - public async Task Vector107WireConformanceToRequestChatImageBase64() + public async Task Vector122WireConformanceToRequestChatImageBase64() { string vectorJson = """ { @@ -7717,7 +8225,7 @@ public async Task Vector107WireConformanceToRequestChatImageBase64() } [Fact] - public async Task Vector108WireConformanceToRequestChatImagePart() + public async Task Vector123WireConformanceToRequestChatImagePart() { string vectorJson = """ { @@ -7783,7 +8291,7 @@ public async Task Vector108WireConformanceToRequestChatImagePart() } [Fact] - public async Task Vector109WireConformanceToRequestChatMultipartContent() + public async Task Vector124WireConformanceToRequestChatMultipartContent() { string vectorJson = """ { @@ -7849,7 +8357,7 @@ public async Task Vector109WireConformanceToRequestChatMultipartContent() } [Fact] - public async Task Vector110WireConformanceToRequestChatSimple() + public async Task Vector125WireConformanceToRequestChatSimple() { string vectorJson = """ { @@ -7913,7 +8421,7 @@ public async Task Vector110WireConformanceToRequestChatSimple() } [Fact] - public async Task Vector111WireConformanceToRequestChatSingleTextOptimized() + public async Task Vector126WireConformanceToRequestChatSingleTextOptimized() { string vectorJson = """ { @@ -7964,7 +8472,7 @@ public async Task Vector111WireConformanceToRequestChatSingleTextOptimized() } [Fact] - public async Task Vector112WireConformanceToRequestChatWithOptions() + public async Task Vector127WireConformanceToRequestChatWithOptions() { string vectorJson = """ { @@ -8034,7 +8542,7 @@ public async Task Vector112WireConformanceToRequestChatWithOptions() } [Fact] - public async Task Vector113WireConformanceToRequestEmbeddingWire() + public async Task Vector128WireConformanceToRequestEmbeddingWire() { string vectorJson = """ { @@ -8080,7 +8588,7 @@ public async Task Vector113WireConformanceToRequestEmbeddingWire() } [Fact] - public async Task Vector114WireConformanceToRequestImageWire() + public async Task Vector129WireConformanceToRequestImageWire() { string vectorJson = """ { @@ -8126,7 +8634,7 @@ public async Task Vector114WireConformanceToRequestImageWire() } [Fact] - public async Task Vector115WireConformanceToRequestKindToJsonTypeMapping() + public async Task Vector130WireConformanceToRequestKindToJsonTypeMapping() { string vectorJson = """ { @@ -8247,7 +8755,7 @@ public async Task Vector115WireConformanceToRequestKindToJsonTypeMapping() } [Fact] - public async Task Vector116WireConformanceToRequestOptionsAdditionalProperties() + public async Task Vector131WireConformanceToRequestOptionsAdditionalProperties() { string vectorJson = """ { @@ -8303,7 +8811,7 @@ public async Task Vector116WireConformanceToRequestOptionsAdditionalProperties() } [Fact] - public async Task Vector117WireConformanceToRequestOptionsMaxCompletionTokens() + public async Task Vector132WireConformanceToRequestOptionsMaxCompletionTokens() { string vectorJson = """ { @@ -8357,7 +8865,7 @@ public async Task Vector117WireConformanceToRequestOptionsMaxCompletionTokens() } [Fact] - public async Task Vector118WireConformanceToRequestOptionsStopSequences() + public async Task Vector133WireConformanceToRequestOptionsStopSequences() { string vectorJson = """ { @@ -8417,7 +8925,7 @@ public async Task Vector118WireConformanceToRequestOptionsStopSequences() } [Fact] - public async Task Vector119WireConformanceToRequestResponsesSimple() + public async Task Vector134WireConformanceToRequestResponsesSimple() { string vectorJson = """ { @@ -8478,7 +8986,7 @@ public async Task Vector119WireConformanceToRequestResponsesSimple() } [Fact] - public async Task Vector120WireConformanceToRequestResponsesStructuredOutput() + public async Task Vector135WireConformanceToRequestResponsesStructuredOutput() { string vectorJson = """ { @@ -8567,7 +9075,7 @@ public async Task Vector120WireConformanceToRequestResponsesStructuredOutput() } [Fact] - public async Task Vector121WireConformanceToRequestResponsesWithTools() + public async Task Vector136WireConformanceToRequestResponsesWithTools() { string vectorJson = """ { @@ -8649,7 +9157,7 @@ public async Task Vector121WireConformanceToRequestResponsesWithTools() } [Fact] - public async Task Vector122WireConformanceToRequestStructuredOutput() + public async Task Vector137WireConformanceToRequestStructuredOutput() { string vectorJson = """ { @@ -8734,7 +9242,7 @@ public async Task Vector122WireConformanceToRequestStructuredOutput() } [Fact] - public async Task Vector123WireConformanceToRequestStructuredOutputNestedOptional() + public async Task Vector138WireConformanceToRequestStructuredOutputNestedOptional() { string vectorJson = """ { @@ -8838,7 +9346,7 @@ public async Task Vector123WireConformanceToRequestStructuredOutputNestedOptiona } [Fact] - public async Task Vector124WireConformanceToRequestToolsBindingsStripped() + public async Task Vector139WireConformanceToRequestToolsBindingsStripped() { string vectorJson = """ { @@ -8932,7 +9440,7 @@ public async Task Vector124WireConformanceToRequestToolsBindingsStripped() } [Fact] - public async Task Vector125WireConformanceToRequestToolsFunctionWire() + public async Task Vector140WireConformanceToRequestToolsFunctionWire() { string vectorJson = """ { @@ -9016,7 +9524,7 @@ public async Task Vector125WireConformanceToRequestToolsFunctionWire() } [Fact] - public async Task Vector126WireConformanceToRequestToolsNullWhenEmpty() + public async Task Vector141WireConformanceToRequestToolsNullWhenEmpty() { string vectorJson = """ { @@ -9067,7 +9575,7 @@ public async Task Vector126WireConformanceToRequestToolsNullWhenEmpty() } [Fact] - public async Task Vector127WireConformanceToRequestToolsStrictMode() + public async Task Vector142WireConformanceToRequestToolsStrictMode() { string vectorJson = """ { diff --git a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs index 57b39cb03..8d3efa5f1 100644 --- a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs +++ b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs @@ -387,6 +387,174 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea if (Canonical(actual.ToJson()) != Canonical("{\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\"}]}")) throw new SeamConformanceException("tools_openapi_load misrouted"); } + { + // vector: trusted_endpoint_builtin_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}")) + throw new SeamConformanceException("trusted_endpoint_builtin_allowed misrouted"); + } + { + // vector: trusted_endpoint_env_endpoint_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}}")) + throw new SeamConformanceException("trusted_endpoint_env_endpoint_allowed misrouted"); + } + { + // vector: trusted_endpoint_env_list_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}}")) + throw new SeamConformanceException("trusted_endpoint_env_list_allowed misrouted"); + } + { + // vector: trusted_endpoint_literal_key_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}}")) + throw new SeamConformanceException("trusted_endpoint_literal_key_allowed misrouted"); + } + { + // vector: trusted_endpoint_option_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}}")) + throw new SeamConformanceException("trusted_endpoint_option_allowed misrouted"); + } + { + // vector: trusted_endpoint_reference_skipped + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}")) + throw new SeamConformanceException("trusted_endpoint_reference_skipped misrouted"); + } + { + // vector: untrusted_endpoint_ambient_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_ambient_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_env_default_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_env_default_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_env_key_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_env_key_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_file_connection_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_file_connection_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_key_with_client_secret_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_key_with_client_secret_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_mismatched_credential_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_mismatched_credential_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_no_key_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_no_key_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_suffix_spoof_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_suffix_spoof_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_wildcard_apex_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_wildcard_apex_rejected: expected an error"); + } } } diff --git a/runtime/go/prompty/model/vector_conformance_test.go b/runtime/go/prompty/model/vector_conformance_test.go index 279217c44..1f9b66c12 100644 --- a/runtime/go/prompty/model/vector_conformance_test.go +++ b/runtime/go/prompty/model/vector_conformance_test.go @@ -811,7 +811,10 @@ func TestVector21LoadConformanceLoadConnectionTypesLoad(t *testing.T) { "endpoint": "https://localhost:8080" } } - } + }, + "trustedEndpoints": [ + "https://localhost:8080" + ] }, "expected": { "model": { @@ -911,7 +914,8 @@ func TestVector24LoadConformanceLoadEnvDefault(t *testing.T) { "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "${env:MISSING_VAR:fallback_value}" + "endpoint": "${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }, @@ -1726,7 +1730,496 @@ func TestVector47LoadConformanceLoadToolsOpenapiLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector48MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { +func TestVector48LoadConformanceLoadTrustedEndpointBuiltinAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_builtin_allowed", + "description": "Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-builtin", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_env_endpoint_allowed", + "description": "An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-endpoint", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:VECTOR_FOUNDRY_ENDPOINT}" + } + } + }, + "env": { + "VECTOR_FOUNDRY_ENDPOINT": "https://host-chosen.example.com" + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://host-chosen.example.com" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector50LoadConformanceLoadTrustedEndpointEnvListAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_env_list_allowed", + "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-list", + "model": { + "id": "llama3", + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "env": { + "PROMPTY_TRUSTED_ENDPOINTS": "https://other.example.com, http://localhost:11434" + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_literal_key_allowed", + "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-literal-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com", + "apiKey": "file-supplied-key" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector52LoadConformanceLoadTrustedEndpointOptionAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_option_allowed", + "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-option", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "trustedEndpoints": [ + "HTTPS://PROXY.Example.com" + ] + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector53LoadConformanceLoadTrustedEndpointReferenceSkipped(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_reference_skipped", + "description": "kind: reference connections carry no endpoint and are not subject to the trust check", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-reference", + "model": { + "id": "gpt-4", + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector54LoadConformanceLoadUntrustedEndpointAmbientRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_ambient_rejected", + "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ambient", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com/api/projects/p" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_env_default_rejected", + "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-default", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:NONEXISTENT:https://attacker.example.com}" + } + } + }, + "env": {} + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_env_key_rejected", + "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": "${env:VECTOR_HOST_KEY}" + } + } + }, + "env": { + "VECTOR_HOST_KEY": "host-secret" + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_file_connection_rejected", + "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-file-conn", + "model": { + "id": "gpt-4", + "connection": "${file:conn.json}" + } + }, + "files": { + "conn.json": { + "kind": "foundry", + "endpoint": "https://attacker.example.com" + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_key_with_client_secret_rejected", + "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-key-client-secret", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "clientSecret": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_mismatched_credential_rejected", + "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-mismatched-credential", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com", + "apiKey": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector60LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_no_key_rejected", + "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-no-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "http://127.0.0.1:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_suffix_spoof_rejected", + "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-spoof", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://good.example.com.attacker.example.com" + } + } + }, + "trustedEndpoints": [ + "https://good.example.com" + ] + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_wildcard_apex_rejected", + "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-apex", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://myres.openai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector63MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorJSON := `{ "name": "clear_by_category_and_all", "description": "Clear removes only the requested category, or all memories when no category is supplied.", @@ -1779,7 +2272,7 @@ func TestVector48MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector49MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { +func TestVector64MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { vectorJSON := `{ "name": "format_core_and_recall_results", "description": "Formatting injects only core memories into system prompts and formats recall results with category and tags.", @@ -1820,7 +2313,7 @@ func TestVector49MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector50MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { +func TestVector65MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { vectorJSON := `{ "name": "recall_empty_query_and_limit", "description": "Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.", @@ -1874,7 +2367,7 @@ func TestVector50MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector51MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { +func TestVector66MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { vectorJSON := `{ "name": "recall_ranks_by_weighted_score", "description": "Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.", @@ -1932,7 +2425,7 @@ func TestVector51MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { +func TestVector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { vectorJSON := `{ "name": "recall_score_ties_preserve_insertion_order", "description": "Recall preserves insertion order for memories with identical scores.", @@ -1989,7 +2482,7 @@ func TestVector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector53MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { +func TestVector68MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorJSON := `{ "name": "recall_tags_and_core_boost", "description": "Recall gives tag matches higher weight than content matches and boosts matching core memories.", @@ -2049,7 +2542,7 @@ func TestVector53MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { +func TestVector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { vectorJSON := `{ "name": "recall_tokenization_punctuation_unicode_and_dedup", "description": "Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.", @@ -2109,7 +2602,7 @@ func TestVector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAnd vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector55MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { +func TestVector70MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_by_tags", "description": "Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.", @@ -2172,7 +2665,7 @@ func TestVector55MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { +func TestVector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_missing_and_empty_tags", "description": "Remember treats missing and empty tag lists as equivalent for core-memory deduplication.", @@ -2227,7 +2720,7 @@ func TestVector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector57MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { +func TestVector72MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { vectorJSON := `{ "name": "remember_evicts_archival_first", "description": "Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.", @@ -2287,7 +2780,7 @@ func TestVector57MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing. vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector58MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { +func TestVector73MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorJSON := `{ "name": "snapshot_port_round_trip", "description": "A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.", @@ -2326,7 +2819,7 @@ func TestVector58MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector59MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { +func TestVector74MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { vectorJSON := `{ "name": "update_index_out_of_range_error", "description": "Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.", @@ -2359,7 +2852,7 @@ func TestVector59MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector60TurnConformanceReplayMaxIterations(t *testing.T) { +func TestVector75TurnConformanceReplayMaxIterations(t *testing.T) { vectorJSON := `{ "name": "max_iterations", "stage": "replay", @@ -2395,7 +2888,7 @@ func TestVector60TurnConformanceReplayMaxIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector61TurnConformanceReplayNoTool(t *testing.T) { +func TestVector76TurnConformanceReplayNoTool(t *testing.T) { vectorJSON := `{ "name": "no_tool", "stage": "replay", @@ -2427,7 +2920,7 @@ func TestVector61TurnConformanceReplayNoTool(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector62TurnConformanceReplayPermissionDenied(t *testing.T) { +func TestVector77TurnConformanceReplayPermissionDenied(t *testing.T) { vectorJSON := `{ "name": "permission_denied", "stage": "replay", @@ -2462,7 +2955,7 @@ func TestVector62TurnConformanceReplayPermissionDenied(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector63TurnConformanceReplayToolFailure(t *testing.T) { +func TestVector78TurnConformanceReplayToolFailure(t *testing.T) { vectorJSON := `{ "name": "tool_failure", "stage": "replay", @@ -2499,7 +2992,7 @@ func TestVector63TurnConformanceReplayToolFailure(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector64TurnConformanceReplayToolSuccess(t *testing.T) { +func TestVector79TurnConformanceReplayToolSuccess(t *testing.T) { vectorJSON := `{ "name": "tool_success", "stage": "replay", @@ -2536,7 +3029,7 @@ func TestVector64TurnConformanceReplayToolSuccess(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector65TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { +func TestVector80TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorJSON := `{ "name": "assistant_tool_calls_metadata", "description": "When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.", @@ -2689,7 +3182,7 @@ func TestVector65TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector66TurnConformanceRunAsyncToolFunction(t *testing.T) { +func TestVector81TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorJSON := `{ "name": "async_tool_function", "description": "Agent loop correctly handles async tool functions", @@ -2800,7 +3293,7 @@ func TestVector66TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector67TurnConformanceRunBindingsInjected(t *testing.T) { +func TestVector82TurnConformanceRunBindingsInjected(t *testing.T) { vectorJSON := `{ "name": "bindings_injected", "description": "Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.", @@ -2931,7 +3424,7 @@ func TestVector67TurnConformanceRunBindingsInjected(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector68TurnConformanceRunCancellationBeforeLlm(t *testing.T) { +func TestVector83TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorJSON := `{ "name": "cancellation_before_llm", "description": "§13.2 Cancellation — Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.", @@ -2993,7 +3486,7 @@ func TestVector68TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector69TurnConformanceRunCancellationBetweenIterations(t *testing.T) { +func TestVector84TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_iterations", "description": "§13.2 Cancellation — Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.", @@ -3124,7 +3617,7 @@ func TestVector69TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector70TurnConformanceRunCancellationBetweenTools(t *testing.T) { +func TestVector85TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_tools", "description": "§13.2 Cancellation — LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.", @@ -3264,7 +3757,7 @@ func TestVector70TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector71TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { +func TestVector86TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorJSON := `{ "name": "context_no_trim_when_fits", "description": "§13.3 Context Window — Messages fit within the context budget. No trimming occurs and all messages are preserved.", @@ -3336,7 +3829,7 @@ func TestVector71TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector72TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { +func TestVector87TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { vectorJSON := `{ "name": "context_preserves_system_messages", "description": "§13.3 Context Window — Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.", @@ -3444,7 +3937,7 @@ func TestVector72TurnConformanceRunContextPreservesSystemMessages(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector73TurnConformanceRunContextTrimBasic(t *testing.T) { +func TestVector88TurnConformanceRunContextTrimBasic(t *testing.T) { vectorJSON := `{ "name": "context_trim_basic", "description": "§13.3 Context Window — Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.", @@ -3603,7 +4096,7 @@ func TestVector73TurnConformanceRunContextTrimBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector74TurnConformanceRunEmptyToolResult(t *testing.T) { +func TestVector89TurnConformanceRunEmptyToolResult(t *testing.T) { vectorJSON := `{ "name": "empty_tool_result", "description": "Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content — it must not be skipped or filtered out.", @@ -3719,7 +4212,7 @@ func TestVector74TurnConformanceRunEmptyToolResult(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector75TurnConformanceRunEventsBasicToolLoop(t *testing.T) { +func TestVector90TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorJSON := `{ "name": "events_basic_tool_loop", "description": "§13.1 Events — A 2-turn tool-call loop emits the full event lifecycle: status → tool_call_start → tool_result → messages_updated → done.", @@ -3866,7 +4359,7 @@ func TestVector75TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector76TurnConformanceRunEventsErrorLogged(t *testing.T) { +func TestVector91TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorJSON := `{ "name": "events_error_logged", "description": "§13.1 Events — A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.", @@ -3993,7 +4486,7 @@ func TestVector76TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector77TurnConformanceRunEventsNoTools(t *testing.T) { +func TestVector92TurnConformanceRunEventsNoTools(t *testing.T) { vectorJSON := `{ "name": "events_no_tools", "description": "§13.1 Events — Single-turn completion with no tool calls. Only the done event is emitted.", @@ -4072,7 +4565,7 @@ func TestVector77TurnConformanceRunEventsNoTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector78TurnConformanceRunGuardrailAllPass(t *testing.T) { +func TestVector93TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorJSON := `{ "name": "guardrail_all_pass", "description": "§13.4 Guardrails — All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.", @@ -4196,7 +4689,7 @@ func TestVector78TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector79TurnConformanceRunGuardrailInputDeny(t *testing.T) { +func TestVector94TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_input_deny", "description": "§13.4 Guardrails — Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.", @@ -4253,7 +4746,7 @@ func TestVector79TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector80TurnConformanceRunGuardrailOutputDeny(t *testing.T) { +func TestVector95TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_output_deny", "description": "§13.4 Guardrails — Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.", @@ -4332,7 +4825,7 @@ func TestVector80TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector81TurnConformanceRunGuardrailToolDeny(t *testing.T) { +func TestVector96TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_tool_deny", "description": "§13.4 Guardrails — LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.", @@ -4487,7 +4980,7 @@ func TestVector81TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector82TurnConformanceRunMaxIterationsExceeded(t *testing.T) { +func TestVector97TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorJSON := `{ "name": "max_iterations_exceeded", "description": "LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.", @@ -4998,7 +5491,7 @@ func TestVector82TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector83TurnConformanceRunMultiTurnToolCalls(t *testing.T) { +func TestVector98TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorJSON := `{ "name": "multi_turn_tool_calls", "description": "Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.", @@ -5176,7 +5669,7 @@ func TestVector83TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector84TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { +func TestVector99TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorJSON := `{ "name": "multiple_tool_calls_single_turn", "description": "Turn 1: LLM returns two parallel tool calls — get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.", @@ -5358,7 +5851,7 @@ func TestVector84TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector85TurnConformanceRunNoToolCalls(t *testing.T) { +func TestVector100TurnConformanceRunNoToolCalls(t *testing.T) { vectorJSON := `{ "name": "no_tool_calls", "description": "LLM returns content immediately with no tool calls — the agent loop completes in a single iteration.", @@ -5427,7 +5920,7 @@ func TestVector85TurnConformanceRunNoToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector86TurnConformanceRunParallelToolsBasic(t *testing.T) { +func TestVector101TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_basic", "description": "§13.6 Parallel Tools — LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.", @@ -5601,7 +6094,7 @@ func TestVector86TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector87TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { +func TestVector102TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_guardrail_deny", "description": "§13.6 Parallel Tools + §13.4 Guardrails — 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.", @@ -5793,7 +6286,7 @@ func TestVector87TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector88TurnConformanceRunParallelToolsWithToolError(t *testing.T) { +func TestVector103TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_tool_error", "description": "§13.6 Parallel Tools + tool error — 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.", @@ -5967,7 +6460,7 @@ func TestVector88TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector89TurnConformanceRunSingleToolCall(t *testing.T) { +func TestVector104TurnConformanceRunSingleToolCall(t *testing.T) { vectorJSON := `{ "name": "single_tool_call", "description": "Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.", @@ -6115,7 +6608,7 @@ func TestVector89TurnConformanceRunSingleToolCall(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector90TurnConformanceRunSteeringInjectMessage(t *testing.T) { +func TestVector105TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorJSON := `{ "name": "steering_inject_message", "description": "§13.5 Steering — After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.", @@ -6324,7 +6817,7 @@ func TestVector90TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector91TurnConformanceRunSteeringMultipleMessages(t *testing.T) { +func TestVector106TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorJSON := `{ "name": "steering_multiple_messages", "description": "§13.5 Steering — Two steering messages are queued and both injected before iteration 2. Tests atomic drain — both appear in order.", @@ -6495,7 +6988,7 @@ func TestVector91TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector92TurnConformanceRunToolNotRegisteredError(t *testing.T) { +func TestVector107TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorJSON := `{ "name": "tool_not_registered_error", "description": "LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.", @@ -6580,7 +7073,7 @@ func TestVector92TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector93TurnConformanceRunToolResultMessageFormat(t *testing.T) { +func TestVector108TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorJSON := `{ "name": "tool_result_message_format", "description": "After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.", @@ -6702,7 +7195,7 @@ func TestVector93TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector94TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { +func TestVector109TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorJSON := `{ "name": "cancel_before_context", "stage": "turn", @@ -6735,7 +7228,7 @@ func TestVector94TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector95TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { +func TestVector110TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorJSON := `{ "name": "delegated_provider_state", "stage": "turn", @@ -6803,7 +7296,7 @@ func TestVector95TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector96TurnConformanceRunTurnFinalOutput(t *testing.T) { +func TestVector111TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorJSON := `{ "name": "final_output", "stage": "turn", @@ -6849,7 +7342,7 @@ func TestVector96TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector97TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { +func TestVector112TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "memory_context_is_model_visible", "stage": "turn", @@ -6912,7 +7405,7 @@ func TestVector97TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector98TurnConformanceRunTurnOrderedToolRound(t *testing.T) { +func TestVector113TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorJSON := `{ "name": "ordered_tool_round", "stage": "turn", @@ -7004,7 +7497,7 @@ func TestVector98TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector99TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { +func TestVector114TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "permission_denial_is_model_visible", "stage": "turn", @@ -7056,7 +7549,7 @@ func TestVector99TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector100WireConformanceToRequestAnthropicImageFormat(t *testing.T) { +func TestVector115WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorJSON := `{ "name": "anthropic_image_format", "description": "§7.5 — Anthropic: ImagePart uses base64 source block with media_type, not image_url format.", @@ -7127,7 +7620,7 @@ func TestVector100WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector101WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { +func TestVector116WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { vectorJSON := `{ "name": "anthropic_max_tokens_required", "description": "§7.5 — Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).", @@ -7183,7 +7676,7 @@ func TestVector101WireConformanceToRequestAnthropicMaxTokensRequired(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector102WireConformanceToRequestAnthropicOptions(t *testing.T) { +func TestVector117WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorJSON := `{ "name": "anthropic_options", "description": "Anthropic: option names map correctly (topK, stopSequences)", @@ -7253,7 +7746,7 @@ func TestVector102WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector103WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { +func TestVector118WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { vectorJSON := `{ "name": "anthropic_system_separate", "description": "§7.5 — Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.", @@ -7321,7 +7814,7 @@ func TestVector103WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector104WireConformanceToRequestAnthropicToolWire(t *testing.T) { +func TestVector119WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorJSON := `{ "name": "anthropic_tool_wire", "description": "Anthropic: tools use input_schema (not parameters), name at top level", @@ -7407,7 +7900,7 @@ func TestVector104WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector105WireConformanceToRequestChatAudioMp3(t *testing.T) { +func TestVector120WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorJSON := `{ "name": "chat_audio_mp3", "description": "§7.1.2 — AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.", @@ -7466,7 +7959,7 @@ func TestVector105WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector106WireConformanceToRequestChatAudioPart(t *testing.T) { +func TestVector121WireConformanceToRequestChatAudioPart(t *testing.T) { vectorJSON := `{ "name": "chat_audio_part", "description": "§7.1.2 — AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.", @@ -7525,7 +8018,7 @@ func TestVector106WireConformanceToRequestChatAudioPart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector107WireConformanceToRequestChatImageBase64(t *testing.T) { +func TestVector122WireConformanceToRequestChatImageBase64(t *testing.T) { vectorJSON := `{ "name": "chat_image_base64", "description": "Base64 data URI image part preserved as-is in image_url", @@ -7590,7 +8083,7 @@ func TestVector107WireConformanceToRequestChatImageBase64(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector108WireConformanceToRequestChatImagePart(t *testing.T) { +func TestVector123WireConformanceToRequestChatImagePart(t *testing.T) { vectorJSON := `{ "name": "chat_image_part", "description": "Image content part converted to OpenAI image_url format", @@ -7655,7 +8148,7 @@ func TestVector108WireConformanceToRequestChatImagePart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector109WireConformanceToRequestChatMultipartContent(t *testing.T) { +func TestVector124WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorJSON := `{ "name": "chat_multipart_content", "description": "§7.1.1/§7.1.2 — Message with TextPart + ImagePart MUST produce an array of typed content blocks.", @@ -7720,7 +8213,7 @@ func TestVector109WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector110WireConformanceToRequestChatSimple(t *testing.T) { +func TestVector125WireConformanceToRequestChatSimple(t *testing.T) { vectorJSON := `{ "name": "chat_simple", "description": "§7.1.7 — Two messages (system + user), no tools, no options. Minimal chat request.", @@ -7783,7 +8276,7 @@ func TestVector110WireConformanceToRequestChatSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector111WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { +func TestVector126WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { vectorJSON := `{ "name": "chat_single_text_optimized", "description": "§7.1.1 — Message with exactly 1 TextPart MUST produce a plain string content, not an array.", @@ -7833,7 +8326,7 @@ func TestVector111WireConformanceToRequestChatSingleTextOptimized(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector112WireConformanceToRequestChatWithOptions(t *testing.T) { +func TestVector127WireConformanceToRequestChatWithOptions(t *testing.T) { vectorJSON := `{ "name": "chat_with_options", "description": "§7.1.5 — All standard ModelOptions mapped to OpenAI parameter names.", @@ -7902,7 +8395,7 @@ func TestVector112WireConformanceToRequestChatWithOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector113WireConformanceToRequestEmbeddingWire(t *testing.T) { +func TestVector128WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorJSON := `{ "name": "embedding_wire", "description": "§7.2 — Embedding request: text extracted from messages, single string input (not array).", @@ -7947,7 +8440,7 @@ func TestVector113WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector114WireConformanceToRequestImageWire(t *testing.T) { +func TestVector129WireConformanceToRequestImageWire(t *testing.T) { vectorJSON := `{ "name": "image_wire", "description": "§7.3 — Image generation request: prompt extracted from last user message.", @@ -7992,7 +8485,7 @@ func TestVector114WireConformanceToRequestImageWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector115WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { +func TestVector130WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorJSON := `{ "name": "kind_to_json_type_mapping", "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.", @@ -8112,7 +8605,7 @@ func TestVector115WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector116WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { +func TestVector131WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { vectorJSON := `{ "name": "options_additional_properties", "description": "§7.1.5 — additionalProperties from ModelOptions MUST be merged into the request as top-level keys.", @@ -8167,7 +8660,7 @@ func TestVector116WireConformanceToRequestOptionsAdditionalProperties(t *testing vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector117WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { +func TestVector132WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { vectorJSON := `{ "name": "options_max_completion_tokens", "description": "§7.1.5 — maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.", @@ -8220,7 +8713,7 @@ func TestVector117WireConformanceToRequestOptionsMaxCompletionTokens(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector118WireConformanceToRequestOptionsStopSequences(t *testing.T) { +func TestVector133WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorJSON := `{ "name": "options_stop_sequences", "description": "§7.1.5 — stopSequences MUST map to 'stop' in the OpenAI wire format.", @@ -8279,7 +8772,7 @@ func TestVector118WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector119WireConformanceToRequestResponsesSimple(t *testing.T) { +func TestVector134WireConformanceToRequestResponsesSimple(t *testing.T) { vectorJSON := `{ "name": "responses_simple", "description": "Responses API: system becomes instructions, messages become input items", @@ -8339,7 +8832,7 @@ func TestVector119WireConformanceToRequestResponsesSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector120WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { +func TestVector135WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "responses_structured_output", "description": "Responses API: structured output uses text.format.json_schema", @@ -8427,7 +8920,7 @@ func TestVector120WireConformanceToRequestResponsesStructuredOutput(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector121WireConformanceToRequestResponsesWithTools(t *testing.T) { +func TestVector136WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorJSON := `{ "name": "responses_with_tools", "description": "Responses API: tools use flat format (no nested function key)", @@ -8508,7 +9001,7 @@ func TestVector121WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector122WireConformanceToRequestStructuredOutput(t *testing.T) { +func TestVector137WireConformanceToRequestStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "structured_output", "description": "§7.1.6 — outputs converted to response_format with json_schema, strict=true, additionalProperties=false.", @@ -8592,7 +9085,7 @@ func TestVector122WireConformanceToRequestStructuredOutput(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector123WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { +func TestVector138WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { vectorJSON := `{ "name": "structured_output_nested_optional", "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", @@ -8695,7 +9188,7 @@ func TestVector123WireConformanceToRequestStructuredOutputNestedOptional(t *test vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector124WireConformanceToRequestToolsBindingsStripped(t *testing.T) { +func TestVector139WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorJSON := `{ "name": "tools_bindings_stripped", "description": "§7.1.3 — Parameters listed in bindings MUST be stripped from wire tools (properties AND required).", @@ -8788,7 +9281,7 @@ func TestVector124WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector125WireConformanceToRequestToolsFunctionWire(t *testing.T) { +func TestVector140WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorJSON := `{ "name": "tools_function_wire", "description": "§7.1.3 — FunctionTool projected as OpenAI function definition with JSON Schema parameters.", @@ -8871,7 +9364,7 @@ func TestVector125WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector126WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { +func TestVector141WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorJSON := `{ "name": "tools_null_when_empty", "description": "§7.1.3 — When no tools are defined, the tools key MUST be absent from the request entirely.", @@ -8921,7 +9414,7 @@ func TestVector126WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector127WireConformanceToRequestToolsStrictMode(t *testing.T) { +func TestVector142WireConformanceToRequestToolsStrictMode(t *testing.T) { vectorJSON := `{ "name": "tools_strict_mode", "description": "§7.1.3 — FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.", diff --git a/runtime/go/prompty/model/vectorconformance/vector_conformance.go b/runtime/go/prompty/model/vectorconformance/vector_conformance.go index 81a5bbde0..1ea2dca28 100644 --- a/runtime/go/prompty/model/vectorconformance/vector_conformance.go +++ b/runtime/go/prompty/model/vectorconformance/vector_conformance.go @@ -1087,4 +1087,259 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("tools_openapi_load misrouted: got %#v, want %#v", actualValue, expectedValue) } } + // vector: trusted_endpoint_builtin_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_builtin_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_builtin_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_builtin_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_builtin_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_builtin_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_builtin_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_env_endpoint_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_env_endpoint_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_env_endpoint_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_env_endpoint_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_env_endpoint_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_env_endpoint_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_env_endpoint_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_env_list_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_env_list_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_env_list_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_env_list_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_env_list_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_env_list_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_env_list_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_literal_key_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_literal_key_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_literal_key_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_literal_key_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_literal_key_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_literal_key_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_literal_key_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_option_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_option_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_option_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_option_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_option_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_option_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_option_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_reference_skipped + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_reference_skipped: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_reference_skipped: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_reference_skipped: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_reference_skipped: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_reference_skipped: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_reference_skipped misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: untrusted_endpoint_ambient_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_ambient_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_ambient_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_env_default_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_env_default_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_env_default_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_env_key_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_env_key_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_env_key_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_file_connection_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_file_connection_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_file_connection_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_key_with_client_secret_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_key_with_client_secret_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_key_with_client_secret_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_mismatched_credential_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_mismatched_credential_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_mismatched_credential_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_no_key_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_no_key_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_no_key_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_suffix_spoof_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_suffix_spoof_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_suffix_spoof_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_wildcard_apex_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_wildcard_apex_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_wildcard_apex_rejected: expected an error, got nil") + } + } } diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java index ffef595c3..ddd7ae655 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java @@ -246,7 +246,7 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex { // vector: connection_types_load StringBuilder inputJson = new StringBuilder(); - inputJson.append("{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}}"); + inputJson.append("{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"trustedEndpoints\":[\"https://localhost:8080\"]}"); java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); Object source = (Object) input.get("source"); Agent actual = seam.load(source); @@ -285,7 +285,7 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex { // vector: env_default StringBuilder inputJson = new StringBuilder(); - inputJson.append("{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}}"); + inputJson.append("{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\",\"apiKey\":\"literal-key\"}}},\"env\":{}}"); java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); Object source = (Object) input.get("source"); Agent actual = seam.load(source); @@ -600,6 +600,210 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex if (!actual.toJson().equals(TypraJson.stringify(expected))) throw new AssertionError("tools_openapi_load misrouted"); } + { + // vector: trusted_endpoint_builtin_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-builtin\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_builtin_allowed misrouted"); + } + { + // vector: trusted_endpoint_env_endpoint_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_env_endpoint_allowed misrouted"); + } + { + // vector: trusted_endpoint_env_list_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_env_list_allowed misrouted"); + } + { + // vector: trusted_endpoint_literal_key_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_literal_key_allowed misrouted"); + } + { + // vector: trusted_endpoint_option_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_option_allowed misrouted"); + } + { + // vector: trusted_endpoint_reference_skipped + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_reference_skipped misrouted"); + } + { + // vector: untrusted_endpoint_ambient_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_ambient_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_env_default_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_env_default_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_env_key_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_env_key_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_file_connection_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_file_connection_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_key_with_client_secret_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_key_with_client_secret_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_mismatched_credential_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_mismatched_credential_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_no_key_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_no_key_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_suffix_spoof_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_suffix_spoof_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_wildcard_apex_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_wildcard_apex_rejected: expected an error"); + } } } diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java index cd3518f3e..bbd28f68a 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java @@ -206,7 +206,7 @@ private static void vector20LoadConformanceLoadBasicLoad() throws Exception { private static void vector21LoadConformanceLoadConnectionTypesLoad() throws Exception { StringBuilder sb = new StringBuilder(); - sb.append("{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}"); + sb.append("{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"trustedEndpoints\":[\"https://localhost:8080\"]},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } @@ -227,7 +227,7 @@ private static void vector23LoadConformanceLoadEmptyFrontmatterBodyOnly() throws private static void vector24LoadConformanceLoadEnvDefault() throws Exception { StringBuilder sb = new StringBuilder(); - sb.append("{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}"); + sb.append("{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\",\"apiKey\":\"literal-key\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } @@ -393,560 +393,665 @@ private static void vector47LoadConformanceLoadToolsOpenapiLoad() throws Excepti VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector48MemoryConformanceOperateClearByCategoryAndAll() throws Exception { + private static void vector48LoadConformanceLoadTrustedEndpointBuiltinAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_builtin_allowed\",\"description\":\"Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-builtin\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_env_endpoint_allowed\",\"description\":\"An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector50LoadConformanceLoadTrustedEndpointEnvListAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_env_list_allowed\",\"description\":\"PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_literal_key_allowed\",\"description\":\"A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector52LoadConformanceLoadTrustedEndpointOptionAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_option_allowed\",\"description\":\"A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector53LoadConformanceLoadTrustedEndpointReferenceSkipped() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_reference_skipped\",\"description\":\"kind: reference connections carry no endpoint and are not subject to the trust check\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector54LoadConformanceLoadUntrustedEndpointAmbientRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_ambient_rejected\",\"description\":\"A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec \\u00a72.5.1)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector60LoadConformanceLoadUntrustedEndpointNoKeyRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector63MemoryConformanceOperateClearByCategoryAndAll() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector49MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { + private static void vector64MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector50MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { + private static void vector65MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector51MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { + private static void vector66MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { + private static void vector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector53MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { + private static void vector68MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { + private static void vector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\\u00e9\"]},{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\\u00e9\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector55MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { + private static void vector70MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { + private static void vector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector57MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { + private static void vector72MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector58MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { + private static void vector73MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector59MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { + private static void vector74MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector60TurnConformanceReplayMaxIterations() throws Exception { + private static void vector75TurnConformanceReplayMaxIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector61TurnConformanceReplayNoTool() throws Exception { + private static void vector76TurnConformanceReplayNoTool() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector62TurnConformanceReplayPermissionDenied() throws Exception { + private static void vector77TurnConformanceReplayPermissionDenied() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector63TurnConformanceReplayToolFailure() throws Exception { + private static void vector78TurnConformanceReplayToolFailure() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector64TurnConformanceReplayToolSuccess() throws Exception { + private static void vector79TurnConformanceReplayToolSuccess() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector65TurnConformanceRunAssistantToolCallsMetadata() throws Exception { + private static void vector80TurnConformanceRunAssistantToolCallsMetadata() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector66TurnConformanceRunAsyncToolFunction() throws Exception { + private static void vector81TurnConformanceRunAsyncToolFunction() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector67TurnConformanceRunBindingsInjected() throws Exception { + private static void vector82TurnConformanceRunBindingsInjected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\\u00b0C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector68TurnConformanceRunCancellationBeforeLlm() throws Exception { + private static void vector83TurnConformanceRunCancellationBeforeLlm() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_before_llm\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector69TurnConformanceRunCancellationBetweenIterations() throws Exception { + private static void vector84TurnConformanceRunCancellationBetweenIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_iterations\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector70TurnConformanceRunCancellationBetweenTools() throws Exception { + private static void vector85TurnConformanceRunCancellationBetweenTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_tools\",\"description\":\"\\u00a713.2 Cancellation \\u2014 LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector71TurnConformanceRunContextNoTrimWhenFits() throws Exception { + private static void vector86TurnConformanceRunContextNoTrimWhenFits() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector72TurnConformanceRunContextPreservesSystemMessages() throws Exception { + private static void vector87TurnConformanceRunContextPreservesSystemMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_preserves_system_messages\",\"description\":\"\\u00a713.3 Context Window \\u2014 Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \\u2014 they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector73TurnConformanceRunContextTrimBasic() throws Exception { + private static void vector88TurnConformanceRunContextTrimBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_trim_basic\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector74TurnConformanceRunEmptyToolResult() throws Exception { + private static void vector89TurnConformanceRunEmptyToolResult() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \\u2014 it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector75TurnConformanceRunEventsBasicToolLoop() throws Exception { + private static void vector90TurnConformanceRunEventsBasicToolLoop() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_basic_tool_loop\",\"description\":\"\\u00a713.1 Events \\u2014 A 2-turn tool-call loop emits the full event lifecycle: status \\u2192 tool_call_start \\u2192 tool_result \\u2192 messages_updated \\u2192 done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\\u00b0F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector76TurnConformanceRunEventsErrorLogged() throws Exception { + private static void vector91TurnConformanceRunEventsErrorLogged() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_error_logged\",\"description\":\"\\u00a713.1 Events \\u0097 A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \\u0097 this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector77TurnConformanceRunEventsNoTools() throws Exception { + private static void vector92TurnConformanceRunEventsNoTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_no_tools\",\"description\":\"\\u00a713.1 Events \\u2014 Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector78TurnConformanceRunGuardrailAllPass() throws Exception { + private static void vector93TurnConformanceRunGuardrailAllPass() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_all_pass\",\"description\":\"\\u00a713.4 Guardrails \\u2014 All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector79TurnConformanceRunGuardrailInputDeny() throws Exception { + private static void vector94TurnConformanceRunGuardrailInputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_input_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector80TurnConformanceRunGuardrailOutputDeny() throws Exception { + private static void vector95TurnConformanceRunGuardrailOutputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_output_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector81TurnConformanceRunGuardrailToolDeny() throws Exception { + private static void vector96TurnConformanceRunGuardrailToolDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_tool_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \\u2014 its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector82TurnConformanceRunMaxIterationsExceeded() throws Exception { + private static void vector97TurnConformanceRunMaxIterationsExceeded() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\\u00b0F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\\u00b0F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\\u00b0F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\\u00b0F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\\u00b0F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\\u00b0F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\\u00b0F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\\u00b0F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\\u00b0F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector83TurnConformanceRunMultiTurnToolCalls() throws Exception { + private static void vector98TurnConformanceRunMultiTurnToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\\u00b0C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector84TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { + private static void vector99TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \\u2014 get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\\u00b0F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector85TurnConformanceRunNoToolCalls() throws Exception { + private static void vector100TurnConformanceRunNoToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \\u2014 the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector86TurnConformanceRunParallelToolsBasic() throws Exception { + private static void vector101TurnConformanceRunParallelToolsBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_basic\",\"description\":\"\\u00a713.6 Parallel Tools \\u2014 LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector87TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { + private static void vector102TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\\u00a713.6 Parallel Tools + \\u00a713.4 Guardrails \\u2014 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector88TurnConformanceRunParallelToolsWithToolError() throws Exception { + private static void vector103TurnConformanceRunParallelToolsWithToolError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\\u00a713.6 Parallel Tools + tool error \\u2014 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector89TurnConformanceRunSingleToolCall() throws Exception { + private static void vector104TurnConformanceRunSingleToolCall() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector90TurnConformanceRunSteeringInjectMessage() throws Exception { + private static void vector105TurnConformanceRunSteeringInjectMessage() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_inject_message\",\"description\":\"\\u00a713.5 Steering \\u2014 After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector91TurnConformanceRunSteeringMultipleMessages() throws Exception { + private static void vector106TurnConformanceRunSteeringMultipleMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_multiple_messages\",\"description\":\"\\u00a713.5 Steering \\u2014 Two steering messages are queued and both injected before iteration 2. Tests atomic drain \\u2014 both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\\u00b0F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector92TurnConformanceRunToolNotRegisteredError() throws Exception { + private static void vector107TurnConformanceRunToolNotRegisteredError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector93TurnConformanceRunToolResultMessageFormat() throws Exception { + private static void vector108TurnConformanceRunToolResultMessageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\\u00b0F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\\u00b0F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector94TurnConformanceRunTurnCancelBeforeContext() throws Exception { + private static void vector109TurnConformanceRunTurnCancelBeforeContext() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector95TurnConformanceRunTurnDelegatedProviderState() throws Exception { + private static void vector110TurnConformanceRunTurnDelegatedProviderState() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector96TurnConformanceRunTurnFinalOutput() throws Exception { + private static void vector111TurnConformanceRunTurnFinalOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector97TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { + private static void vector112TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector98TurnConformanceRunTurnOrderedToolRound() throws Exception { + private static void vector113TurnConformanceRunTurnOrderedToolRound() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector99TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { + private static void vector114TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector100WireConformanceToRequestAnthropicImageFormat() throws Exception { + private static void vector115WireConformanceToRequestAnthropicImageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_image_format\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector101WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { + private static void vector116WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector102WireConformanceToRequestAnthropicOptions() throws Exception { + private static void vector117WireConformanceToRequestAnthropicOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector103WireConformanceToRequestAnthropicSystemSeparate() throws Exception { + private static void vector118WireConformanceToRequestAnthropicSystemSeparate() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_system_separate\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector104WireConformanceToRequestAnthropicToolWire() throws Exception { + private static void vector119WireConformanceToRequestAnthropicToolWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector105WireConformanceToRequestChatAudioMp3() throws Exception { + private static void vector120WireConformanceToRequestChatAudioMp3() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_mp3\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector106WireConformanceToRequestChatAudioPart() throws Exception { + private static void vector121WireConformanceToRequestChatAudioPart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_part\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector107WireConformanceToRequestChatImageBase64() throws Exception { + private static void vector122WireConformanceToRequestChatImageBase64() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector108WireConformanceToRequestChatImagePart() throws Exception { + private static void vector123WireConformanceToRequestChatImagePart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector109WireConformanceToRequestChatMultipartContent() throws Exception { + private static void vector124WireConformanceToRequestChatMultipartContent() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_multipart_content\",\"description\":\"\\u00a77.1.1/\\u00a77.1.2 \\u2014 Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector110WireConformanceToRequestChatSimple() throws Exception { + private static void vector125WireConformanceToRequestChatSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_simple\",\"description\":\"\\u00a77.1.7 \\u2014 Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector111WireConformanceToRequestChatSingleTextOptimized() throws Exception { + private static void vector126WireConformanceToRequestChatSingleTextOptimized() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_single_text_optimized\",\"description\":\"\\u00a77.1.1 \\u2014 Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector112WireConformanceToRequestChatWithOptions() throws Exception { + private static void vector127WireConformanceToRequestChatWithOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_with_options\",\"description\":\"\\u00a77.1.5 \\u2014 All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector113WireConformanceToRequestEmbeddingWire() throws Exception { + private static void vector128WireConformanceToRequestEmbeddingWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"embedding_wire\",\"description\":\"\\u00a77.2 \\u2014 Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector114WireConformanceToRequestImageWire() throws Exception { + private static void vector129WireConformanceToRequestImageWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"image_wire\",\"description\":\"\\u00a77.3 \\u2014 Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector115WireConformanceToRequestKindToJsonTypeMapping() throws Exception { + private static void vector130WireConformanceToRequestKindToJsonTypeMapping() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\\u00a77.1.4 \\u2014 All Property kind values mapped to JSON Schema types: string\\u2192string, integer\\u2192integer, float\\u2192number, boolean\\u2192boolean, array\\u2192array, object\\u2192object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector116WireConformanceToRequestOptionsAdditionalProperties() throws Exception { + private static void vector131WireConformanceToRequestOptionsAdditionalProperties() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_additional_properties\",\"description\":\"\\u00a77.1.5 \\u2014 additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector117WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { + private static void vector132WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_max_completion_tokens\",\"description\":\"\\u00a77.1.5 \\u2014 maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector118WireConformanceToRequestOptionsStopSequences() throws Exception { + private static void vector133WireConformanceToRequestOptionsStopSequences() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_stop_sequences\",\"description\":\"\\u00a77.1.5 \\u2014 stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector119WireConformanceToRequestResponsesSimple() throws Exception { + private static void vector134WireConformanceToRequestResponsesSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector120WireConformanceToRequestResponsesStructuredOutput() throws Exception { + private static void vector135WireConformanceToRequestResponsesStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector121WireConformanceToRequestResponsesWithTools() throws Exception { + private static void vector136WireConformanceToRequestResponsesWithTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector122WireConformanceToRequestStructuredOutput() throws Exception { + private static void vector137WireConformanceToRequestStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output\",\"description\":\"\\u00a77.1.6 \\u2014 outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector123WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { + private static void vector138WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output_nested_optional\",\"description\":\"\\u00a77.1.4/\\u00a77.1.6 \\u2014 OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector124WireConformanceToRequestToolsBindingsStripped() throws Exception { + private static void vector139WireConformanceToRequestToolsBindingsStripped() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_bindings_stripped\",\"description\":\"\\u00a77.1.3 \\u2014 Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector125WireConformanceToRequestToolsFunctionWire() throws Exception { + private static void vector140WireConformanceToRequestToolsFunctionWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_function_wire\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector126WireConformanceToRequestToolsNullWhenEmpty() throws Exception { + private static void vector141WireConformanceToRequestToolsNullWhenEmpty() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_null_when_empty\",\"description\":\"\\u00a77.1.3 \\u2014 When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector127WireConformanceToRequestToolsStrictMode() throws Exception { + private static void vector142WireConformanceToRequestToolsStrictMode() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_strict_mode\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); @@ -1003,86 +1108,101 @@ public static void run() { failed += runCase("LoadConformance.load:tools_function_load", VectorConformanceTests::vector45LoadConformanceLoadToolsFunctionLoad); failed += runCase("LoadConformance.load:tools_mcp_load", VectorConformanceTests::vector46LoadConformanceLoadToolsMcpLoad); failed += runCase("LoadConformance.load:tools_openapi_load", VectorConformanceTests::vector47LoadConformanceLoadToolsOpenapiLoad); - failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector48MemoryConformanceOperateClearByCategoryAndAll); - failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector49MemoryConformanceOperateFormatCoreAndRecallResults); - failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector50MemoryConformanceOperateRecallEmptyQueryAndLimit); - failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector51MemoryConformanceOperateRecallRanksByWeightedScore); - failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); - failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector53MemoryConformanceOperateRecallTagsAndCoreBoost); - failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); - failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector55MemoryConformanceOperateRememberDedupsCoreByTags); - failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); - failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector57MemoryConformanceOperateRememberEvictsArchivalFirst); - failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector58MemoryConformanceOperateSnapshotPortRoundTrip); - failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector59MemoryConformanceOperateUpdateIndexOutOfRangeError); - failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector60TurnConformanceReplayMaxIterations); - failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector61TurnConformanceReplayNoTool); - failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector62TurnConformanceReplayPermissionDenied); - failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector63TurnConformanceReplayToolFailure); - failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector64TurnConformanceReplayToolSuccess); - failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector65TurnConformanceRunAssistantToolCallsMetadata); - failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector66TurnConformanceRunAsyncToolFunction); - failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector67TurnConformanceRunBindingsInjected); - failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector68TurnConformanceRunCancellationBeforeLlm); - failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector69TurnConformanceRunCancellationBetweenIterations); - failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector70TurnConformanceRunCancellationBetweenTools); - failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector71TurnConformanceRunContextNoTrimWhenFits); - failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector72TurnConformanceRunContextPreservesSystemMessages); - failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector73TurnConformanceRunContextTrimBasic); - failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector74TurnConformanceRunEmptyToolResult); - failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector75TurnConformanceRunEventsBasicToolLoop); - failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector76TurnConformanceRunEventsErrorLogged); - failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector77TurnConformanceRunEventsNoTools); - failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector78TurnConformanceRunGuardrailAllPass); - failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector79TurnConformanceRunGuardrailInputDeny); - failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector80TurnConformanceRunGuardrailOutputDeny); - failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector81TurnConformanceRunGuardrailToolDeny); - failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector82TurnConformanceRunMaxIterationsExceeded); - failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector83TurnConformanceRunMultiTurnToolCalls); - failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector84TurnConformanceRunMultipleToolCallsSingleTurn); - failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector85TurnConformanceRunNoToolCalls); - failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector86TurnConformanceRunParallelToolsBasic); - failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector87TurnConformanceRunParallelToolsWithGuardrailDeny); - failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector88TurnConformanceRunParallelToolsWithToolError); - failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector89TurnConformanceRunSingleToolCall); - failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector90TurnConformanceRunSteeringInjectMessage); - failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector91TurnConformanceRunSteeringMultipleMessages); - failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector92TurnConformanceRunToolNotRegisteredError); - failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector93TurnConformanceRunToolResultMessageFormat); - failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector94TurnConformanceRunTurnCancelBeforeContext); - failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector95TurnConformanceRunTurnDelegatedProviderState); - failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector96TurnConformanceRunTurnFinalOutput); - failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector97TurnConformanceRunTurnMemoryContextIsModelVisible); - failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector98TurnConformanceRunTurnOrderedToolRound); - failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector99TurnConformanceRunTurnPermissionDenialIsModelVisible); - failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector100WireConformanceToRequestAnthropicImageFormat); - failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector101WireConformanceToRequestAnthropicMaxTokensRequired); - failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector102WireConformanceToRequestAnthropicOptions); - failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector103WireConformanceToRequestAnthropicSystemSeparate); - failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector104WireConformanceToRequestAnthropicToolWire); - failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector105WireConformanceToRequestChatAudioMp3); - failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector106WireConformanceToRequestChatAudioPart); - failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector107WireConformanceToRequestChatImageBase64); - failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector108WireConformanceToRequestChatImagePart); - failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector109WireConformanceToRequestChatMultipartContent); - failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector110WireConformanceToRequestChatSimple); - failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector111WireConformanceToRequestChatSingleTextOptimized); - failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector112WireConformanceToRequestChatWithOptions); - failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector113WireConformanceToRequestEmbeddingWire); - failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector114WireConformanceToRequestImageWire); - failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector115WireConformanceToRequestKindToJsonTypeMapping); - failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector116WireConformanceToRequestOptionsAdditionalProperties); - failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector117WireConformanceToRequestOptionsMaxCompletionTokens); - failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector118WireConformanceToRequestOptionsStopSequences); - failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector119WireConformanceToRequestResponsesSimple); - failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector120WireConformanceToRequestResponsesStructuredOutput); - failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector121WireConformanceToRequestResponsesWithTools); - failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector122WireConformanceToRequestStructuredOutput); - failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector123WireConformanceToRequestStructuredOutputNestedOptional); - failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector124WireConformanceToRequestToolsBindingsStripped); - failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector125WireConformanceToRequestToolsFunctionWire); - failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector126WireConformanceToRequestToolsNullWhenEmpty); - failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector127WireConformanceToRequestToolsStrictMode); + failed += runCase("LoadConformance.load:trusted_endpoint_builtin_allowed", VectorConformanceTests::vector48LoadConformanceLoadTrustedEndpointBuiltinAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_env_endpoint_allowed", VectorConformanceTests::vector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_env_list_allowed", VectorConformanceTests::vector50LoadConformanceLoadTrustedEndpointEnvListAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_literal_key_allowed", VectorConformanceTests::vector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_option_allowed", VectorConformanceTests::vector52LoadConformanceLoadTrustedEndpointOptionAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_reference_skipped", VectorConformanceTests::vector53LoadConformanceLoadTrustedEndpointReferenceSkipped); + failed += runCase("LoadConformance.load:untrusted_endpoint_ambient_rejected", VectorConformanceTests::vector54LoadConformanceLoadUntrustedEndpointAmbientRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_env_default_rejected", VectorConformanceTests::vector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_env_key_rejected", VectorConformanceTests::vector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_file_connection_rejected", VectorConformanceTests::vector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", VectorConformanceTests::vector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", VectorConformanceTests::vector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_no_key_rejected", VectorConformanceTests::vector60LoadConformanceLoadUntrustedEndpointNoKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", VectorConformanceTests::vector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", VectorConformanceTests::vector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected); + failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector63MemoryConformanceOperateClearByCategoryAndAll); + failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector64MemoryConformanceOperateFormatCoreAndRecallResults); + failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector65MemoryConformanceOperateRecallEmptyQueryAndLimit); + failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector66MemoryConformanceOperateRecallRanksByWeightedScore); + failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); + failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector68MemoryConformanceOperateRecallTagsAndCoreBoost); + failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); + failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector70MemoryConformanceOperateRememberDedupsCoreByTags); + failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); + failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector72MemoryConformanceOperateRememberEvictsArchivalFirst); + failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector73MemoryConformanceOperateSnapshotPortRoundTrip); + failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector74MemoryConformanceOperateUpdateIndexOutOfRangeError); + failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector75TurnConformanceReplayMaxIterations); + failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector76TurnConformanceReplayNoTool); + failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector77TurnConformanceReplayPermissionDenied); + failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector78TurnConformanceReplayToolFailure); + failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector79TurnConformanceReplayToolSuccess); + failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector80TurnConformanceRunAssistantToolCallsMetadata); + failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector81TurnConformanceRunAsyncToolFunction); + failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector82TurnConformanceRunBindingsInjected); + failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector83TurnConformanceRunCancellationBeforeLlm); + failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector84TurnConformanceRunCancellationBetweenIterations); + failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector85TurnConformanceRunCancellationBetweenTools); + failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector86TurnConformanceRunContextNoTrimWhenFits); + failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector87TurnConformanceRunContextPreservesSystemMessages); + failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector88TurnConformanceRunContextTrimBasic); + failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector89TurnConformanceRunEmptyToolResult); + failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector90TurnConformanceRunEventsBasicToolLoop); + failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector91TurnConformanceRunEventsErrorLogged); + failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector92TurnConformanceRunEventsNoTools); + failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector93TurnConformanceRunGuardrailAllPass); + failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector94TurnConformanceRunGuardrailInputDeny); + failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector95TurnConformanceRunGuardrailOutputDeny); + failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector96TurnConformanceRunGuardrailToolDeny); + failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector97TurnConformanceRunMaxIterationsExceeded); + failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector98TurnConformanceRunMultiTurnToolCalls); + failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector99TurnConformanceRunMultipleToolCallsSingleTurn); + failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector100TurnConformanceRunNoToolCalls); + failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector101TurnConformanceRunParallelToolsBasic); + failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector102TurnConformanceRunParallelToolsWithGuardrailDeny); + failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector103TurnConformanceRunParallelToolsWithToolError); + failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector104TurnConformanceRunSingleToolCall); + failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector105TurnConformanceRunSteeringInjectMessage); + failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector106TurnConformanceRunSteeringMultipleMessages); + failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector107TurnConformanceRunToolNotRegisteredError); + failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector108TurnConformanceRunToolResultMessageFormat); + failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector109TurnConformanceRunTurnCancelBeforeContext); + failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector110TurnConformanceRunTurnDelegatedProviderState); + failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector111TurnConformanceRunTurnFinalOutput); + failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector112TurnConformanceRunTurnMemoryContextIsModelVisible); + failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector113TurnConformanceRunTurnOrderedToolRound); + failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector114TurnConformanceRunTurnPermissionDenialIsModelVisible); + failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector115WireConformanceToRequestAnthropicImageFormat); + failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector116WireConformanceToRequestAnthropicMaxTokensRequired); + failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector117WireConformanceToRequestAnthropicOptions); + failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector118WireConformanceToRequestAnthropicSystemSeparate); + failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector119WireConformanceToRequestAnthropicToolWire); + failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector120WireConformanceToRequestChatAudioMp3); + failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector121WireConformanceToRequestChatAudioPart); + failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector122WireConformanceToRequestChatImageBase64); + failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector123WireConformanceToRequestChatImagePart); + failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector124WireConformanceToRequestChatMultipartContent); + failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector125WireConformanceToRequestChatSimple); + failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector126WireConformanceToRequestChatSingleTextOptimized); + failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector127WireConformanceToRequestChatWithOptions); + failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector128WireConformanceToRequestEmbeddingWire); + failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector129WireConformanceToRequestImageWire); + failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector130WireConformanceToRequestKindToJsonTypeMapping); + failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector131WireConformanceToRequestOptionsAdditionalProperties); + failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector132WireConformanceToRequestOptionsMaxCompletionTokens); + failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector133WireConformanceToRequestOptionsStopSequences); + failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector134WireConformanceToRequestResponsesSimple); + failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector135WireConformanceToRequestResponsesStructuredOutput); + failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector136WireConformanceToRequestResponsesWithTools); + failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector137WireConformanceToRequestStructuredOutput); + failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector138WireConformanceToRequestStructuredOutputNestedOptional); + failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector139WireConformanceToRequestToolsBindingsStripped); + failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector140WireConformanceToRequestToolsFunctionWire); + failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector141WireConformanceToRequestToolsNullWhenEmpty); + failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector142WireConformanceToRequestToolsStrictMode); if (failed > 0) { throw new AssertionError(failed + " @vector conformance failure(s)"); } diff --git a/runtime/python/prompty/prompty/model/vector_conformance.py b/runtime/python/prompty/prompty/model/vector_conformance.py index 27e45e03b..eb796a5b3 100644 --- a/runtime/python/prompty/prompty/model/vector_conformance.py +++ b/runtime/python/prompty/prompty/model/vector_conformance.py @@ -459,3 +459,151 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( "tools_openapi_load misrouted" ) + # vector: trusted_endpoint_builtin_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://myres.services.ai.azure.com/api/projects/p"}}}' + ) + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_builtin_allowed misrouted" + ) + # vector: trusted_endpoint_env_endpoint_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"foundry","endpoint":"https://host-chosen.example.com"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_env_endpoint_allowed misrouted" + ) + # vector: trusted_endpoint_env_list_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"key","endpoint":"http://localhost:11434/v1"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_env_list_allowed misrouted" + ) + # vector: trusted_endpoint_literal_key_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"key","endpoint":"https://anywhere.example.com"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_literal_key_allowed misrouted" + ) + # vector: trusted_endpoint_option_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://proxy.example.com:443/openai/v1"}}}' + ) + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_option_allowed misrouted" + ) + # vector: trusted_endpoint_reference_skipped + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"reference","name":"my-connection"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_reference_skipped misrouted" + ) + # vector: untrusted_endpoint_ambient_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_ambient_rejected: expected an error" + # vector: untrusted_endpoint_env_default_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_env_default_rejected: expected an error" + # vector: untrusted_endpoint_env_key_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_env_key_rejected: expected an error" + # vector: untrusted_endpoint_file_connection_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_file_connection_rejected: expected an error" + # vector: untrusted_endpoint_key_with_client_secret_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_key_with_client_secret_rejected: expected an error" + # vector: untrusted_endpoint_mismatched_credential_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_mismatched_credential_rejected: expected an error" + # vector: untrusted_endpoint_no_key_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_no_key_rejected: expected an error" + # vector: untrusted_endpoint_suffix_spoof_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_suffix_spoof_rejected: expected an error" + # vector: untrusted_endpoint_wildcard_apex_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_wildcard_apex_rejected: expected an error" diff --git a/runtime/python/prompty/tests/model/test_vector_conformance.py b/runtime/python/prompty/tests/model/test_vector_conformance.py index d01cf8642..5c218561d 100644 --- a/runtime/python/prompty/tests/model/test_vector_conformance.py +++ b/runtime/python/prompty/tests/model/test_vector_conformance.py @@ -785,7 +785,10 @@ async def test_vector_21_loadconformance_load_connection_types_load(): "endpoint": "https://localhost:8080" } } - } + }, + "trustedEndpoints": [ + "https://localhost:8080" + ] }, "expected": { "model": { @@ -882,7 +885,8 @@ async def test_vector_24_loadconformance_load_env_default(): "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "${env:MISSING_VAR:fallback_value}" + "endpoint": "${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }, @@ -1697,7 +1701,481 @@ async def test_vector_47_loadconformance_load_tools_openapi_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_48_memoryconformance_operate_clear_by_category_and_all(): +async def test_vector_48_loadconformance_load_trusted_endpoint_builtin_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_builtin_allowed", + "description": "Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-builtin", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_49_loadconformance_load_trusted_endpoint_env_endpoint_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_env_endpoint_allowed", + "description": "An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-endpoint", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:VECTOR_FOUNDRY_ENDPOINT}" + } + } + }, + "env": { + "VECTOR_FOUNDRY_ENDPOINT": "https://host-chosen.example.com" + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://host-chosen.example.com" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_50_loadconformance_load_trusted_endpoint_env_list_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_env_list_allowed", + "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-list", + "model": { + "id": "llama3", + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "env": { + "PROMPTY_TRUSTED_ENDPOINTS": "https://other.example.com, http://localhost:11434" + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_51_loadconformance_load_trusted_endpoint_literal_key_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_literal_key_allowed", + "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-literal-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com", + "apiKey": "file-supplied-key" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_52_loadconformance_load_trusted_endpoint_option_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_option_allowed", + "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-option", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "trustedEndpoints": [ + "HTTPS://PROXY.Example.com" + ] + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_53_loadconformance_load_trusted_endpoint_reference_skipped(): + vector_json = r""" + { + "name": "trusted_endpoint_reference_skipped", + "description": "kind: reference connections carry no endpoint and are not subject to the trust check", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-reference", + "model": { + "id": "gpt-4", + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_54_loadconformance_load_untrusted_endpoint_ambient_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_ambient_rejected", + "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ambient", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com/api/projects/p" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_55_loadconformance_load_untrusted_endpoint_env_default_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_env_default_rejected", + "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-default", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:NONEXISTENT:https://attacker.example.com}" + } + } + }, + "env": {} + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_56_loadconformance_load_untrusted_endpoint_env_key_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_env_key_rejected", + "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": "${env:VECTOR_HOST_KEY}" + } + } + }, + "env": { + "VECTOR_HOST_KEY": "host-secret" + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_57_loadconformance_load_untrusted_endpoint_file_connection_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_file_connection_rejected", + "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-file-conn", + "model": { + "id": "gpt-4", + "connection": "${file:conn.json}" + } + }, + "files": { + "conn.json": { + "kind": "foundry", + "endpoint": "https://attacker.example.com" + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_58_loadconformance_load_untrusted_endpoint_key_with_client_secret_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_key_with_client_secret_rejected", + "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-key-client-secret", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "clientSecret": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_59_loadconformance_load_untrusted_endpoint_mismatched_credential_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_mismatched_credential_rejected", + "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-mismatched-credential", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com", + "apiKey": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_60_loadconformance_load_untrusted_endpoint_no_key_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_no_key_rejected", + "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-no-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "http://127.0.0.1:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_61_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_suffix_spoof_rejected", + "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-spoof", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://good.example.com.attacker.example.com" + } + } + }, + "trustedEndpoints": [ + "https://good.example.com" + ] + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_62_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_wildcard_apex_rejected", + "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-apex", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://myres.openai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_63_memoryconformance_operate_clear_by_category_and_all(): vector_json = r""" { "name": "clear_by_category_and_all", @@ -1749,7 +2227,7 @@ async def test_vector_48_memoryconformance_operate_clear_by_category_and_all(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_49_memoryconformance_operate_format_core_and_recall_results(): +async def test_vector_64_memoryconformance_operate_format_core_and_recall_results(): vector_json = r""" { "name": "format_core_and_recall_results", @@ -1789,7 +2267,7 @@ async def test_vector_49_memoryconformance_operate_format_core_and_recall_result await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_50_memoryconformance_operate_recall_empty_query_and_limit(): +async def test_vector_65_memoryconformance_operate_recall_empty_query_and_limit(): vector_json = r""" { "name": "recall_empty_query_and_limit", @@ -1842,7 +2320,7 @@ async def test_vector_50_memoryconformance_operate_recall_empty_query_and_limit( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_51_memoryconformance_operate_recall_ranks_by_weighted_score(): +async def test_vector_66_memoryconformance_operate_recall_ranks_by_weighted_score(): vector_json = r""" { "name": "recall_ranks_by_weighted_score", @@ -1899,7 +2377,7 @@ async def test_vector_51_memoryconformance_operate_recall_ranks_by_weighted_scor await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_52_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): +async def test_vector_67_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): vector_json = r""" { "name": "recall_score_ties_preserve_insertion_order", @@ -1955,7 +2433,7 @@ async def test_vector_52_memoryconformance_operate_recall_score_ties_preserve_in await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_53_memoryconformance_operate_recall_tags_and_core_boost(): +async def test_vector_68_memoryconformance_operate_recall_tags_and_core_boost(): vector_json = r""" { "name": "recall_tags_and_core_boost", @@ -2014,7 +2492,7 @@ async def test_vector_53_memoryconformance_operate_recall_tags_and_core_boost(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_54_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): +async def test_vector_69_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): vector_json = r""" { "name": "recall_tokenization_punctuation_unicode_and_dedup", @@ -2073,7 +2551,7 @@ async def test_vector_54_memoryconformance_operate_recall_tokenization_punctuati await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_55_memoryconformance_operate_remember_dedups_core_by_tags(): +async def test_vector_70_memoryconformance_operate_remember_dedups_core_by_tags(): vector_json = r""" { "name": "remember_dedups_core_by_tags", @@ -2135,7 +2613,7 @@ async def test_vector_55_memoryconformance_operate_remember_dedups_core_by_tags( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_56_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): +async def test_vector_71_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): vector_json = r""" { "name": "remember_dedups_core_missing_and_empty_tags", @@ -2189,7 +2667,7 @@ async def test_vector_56_memoryconformance_operate_remember_dedups_core_missing_ await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_57_memoryconformance_operate_remember_evicts_archival_first(): +async def test_vector_72_memoryconformance_operate_remember_evicts_archival_first(): vector_json = r""" { "name": "remember_evicts_archival_first", @@ -2248,7 +2726,7 @@ async def test_vector_57_memoryconformance_operate_remember_evicts_archival_firs await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_58_memoryconformance_operate_snapshot_port_round_trip(): +async def test_vector_73_memoryconformance_operate_snapshot_port_round_trip(): vector_json = r""" { "name": "snapshot_port_round_trip", @@ -2286,7 +2764,7 @@ async def test_vector_58_memoryconformance_operate_snapshot_port_round_trip(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_59_memoryconformance_operate_update_index_out_of_range_error(): +async def test_vector_74_memoryconformance_operate_update_index_out_of_range_error(): vector_json = r""" { "name": "update_index_out_of_range_error", @@ -2318,7 +2796,7 @@ async def test_vector_59_memoryconformance_operate_update_index_out_of_range_err await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_60_turnconformance_replay_max_iterations(): +async def test_vector_75_turnconformance_replay_max_iterations(): vector_json = r""" { "name": "max_iterations", @@ -2353,7 +2831,7 @@ async def test_vector_60_turnconformance_replay_max_iterations(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_61_turnconformance_replay_no_tool(): +async def test_vector_76_turnconformance_replay_no_tool(): vector_json = r""" { "name": "no_tool", @@ -2384,7 +2862,7 @@ async def test_vector_61_turnconformance_replay_no_tool(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_62_turnconformance_replay_permission_denied(): +async def test_vector_77_turnconformance_replay_permission_denied(): vector_json = r""" { "name": "permission_denied", @@ -2418,7 +2896,7 @@ async def test_vector_62_turnconformance_replay_permission_denied(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_63_turnconformance_replay_tool_failure(): +async def test_vector_78_turnconformance_replay_tool_failure(): vector_json = r""" { "name": "tool_failure", @@ -2454,7 +2932,7 @@ async def test_vector_63_turnconformance_replay_tool_failure(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_64_turnconformance_replay_tool_success(): +async def test_vector_79_turnconformance_replay_tool_success(): vector_json = r""" { "name": "tool_success", @@ -2490,7 +2968,7 @@ async def test_vector_64_turnconformance_replay_tool_success(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_65_turnconformance_run_assistant_tool_calls_metadata(): +async def test_vector_80_turnconformance_run_assistant_tool_calls_metadata(): vector_json = r""" { "name": "assistant_tool_calls_metadata", @@ -2642,7 +3120,7 @@ async def test_vector_65_turnconformance_run_assistant_tool_calls_metadata(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_66_turnconformance_run_async_tool_function(): +async def test_vector_81_turnconformance_run_async_tool_function(): vector_json = r""" { "name": "async_tool_function", @@ -2752,7 +3230,7 @@ async def test_vector_66_turnconformance_run_async_tool_function(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_67_turnconformance_run_bindings_injected(): +async def test_vector_82_turnconformance_run_bindings_injected(): vector_json = r""" { "name": "bindings_injected", @@ -2882,7 +3360,7 @@ async def test_vector_67_turnconformance_run_bindings_injected(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_68_turnconformance_run_cancellation_before_llm(): +async def test_vector_83_turnconformance_run_cancellation_before_llm(): vector_json = r""" { "name": "cancellation_before_llm", @@ -2943,7 +3421,7 @@ async def test_vector_68_turnconformance_run_cancellation_before_llm(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_69_turnconformance_run_cancellation_between_iterations(): +async def test_vector_84_turnconformance_run_cancellation_between_iterations(): vector_json = r""" { "name": "cancellation_between_iterations", @@ -3073,7 +3551,7 @@ async def test_vector_69_turnconformance_run_cancellation_between_iterations(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_70_turnconformance_run_cancellation_between_tools(): +async def test_vector_85_turnconformance_run_cancellation_between_tools(): vector_json = r""" { "name": "cancellation_between_tools", @@ -3212,7 +3690,7 @@ async def test_vector_70_turnconformance_run_cancellation_between_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_71_turnconformance_run_context_no_trim_when_fits(): +async def test_vector_86_turnconformance_run_context_no_trim_when_fits(): vector_json = r""" { "name": "context_no_trim_when_fits", @@ -3283,7 +3761,7 @@ async def test_vector_71_turnconformance_run_context_no_trim_when_fits(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_72_turnconformance_run_context_preserves_system_messages(): +async def test_vector_87_turnconformance_run_context_preserves_system_messages(): vector_json = r""" { "name": "context_preserves_system_messages", @@ -3390,7 +3868,7 @@ async def test_vector_72_turnconformance_run_context_preserves_system_messages() await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_73_turnconformance_run_context_trim_basic(): +async def test_vector_88_turnconformance_run_context_trim_basic(): vector_json = r""" { "name": "context_trim_basic", @@ -3548,7 +4026,7 @@ async def test_vector_73_turnconformance_run_context_trim_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_74_turnconformance_run_empty_tool_result(): +async def test_vector_89_turnconformance_run_empty_tool_result(): vector_json = r""" { "name": "empty_tool_result", @@ -3663,7 +4141,7 @@ async def test_vector_74_turnconformance_run_empty_tool_result(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_75_turnconformance_run_events_basic_tool_loop(): +async def test_vector_90_turnconformance_run_events_basic_tool_loop(): vector_json = r""" { "name": "events_basic_tool_loop", @@ -3809,7 +4287,7 @@ async def test_vector_75_turnconformance_run_events_basic_tool_loop(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_76_turnconformance_run_events_error_logged(): +async def test_vector_91_turnconformance_run_events_error_logged(): vector_json = r""" { "name": "events_error_logged", @@ -3935,7 +4413,7 @@ async def test_vector_76_turnconformance_run_events_error_logged(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_77_turnconformance_run_events_no_tools(): +async def test_vector_92_turnconformance_run_events_no_tools(): vector_json = r""" { "name": "events_no_tools", @@ -4013,7 +4491,7 @@ async def test_vector_77_turnconformance_run_events_no_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_78_turnconformance_run_guardrail_all_pass(): +async def test_vector_93_turnconformance_run_guardrail_all_pass(): vector_json = r""" { "name": "guardrail_all_pass", @@ -4136,7 +4614,7 @@ async def test_vector_78_turnconformance_run_guardrail_all_pass(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_79_turnconformance_run_guardrail_input_deny(): +async def test_vector_94_turnconformance_run_guardrail_input_deny(): vector_json = r""" { "name": "guardrail_input_deny", @@ -4192,7 +4670,7 @@ async def test_vector_79_turnconformance_run_guardrail_input_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_80_turnconformance_run_guardrail_output_deny(): +async def test_vector_95_turnconformance_run_guardrail_output_deny(): vector_json = r""" { "name": "guardrail_output_deny", @@ -4270,7 +4748,7 @@ async def test_vector_80_turnconformance_run_guardrail_output_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_81_turnconformance_run_guardrail_tool_deny(): +async def test_vector_96_turnconformance_run_guardrail_tool_deny(): vector_json = r""" { "name": "guardrail_tool_deny", @@ -4424,7 +4902,7 @@ async def test_vector_81_turnconformance_run_guardrail_tool_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_82_turnconformance_run_max_iterations_exceeded(): +async def test_vector_97_turnconformance_run_max_iterations_exceeded(): vector_json = r""" { "name": "max_iterations_exceeded", @@ -4934,7 +5412,7 @@ async def test_vector_82_turnconformance_run_max_iterations_exceeded(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_83_turnconformance_run_multi_turn_tool_calls(): +async def test_vector_98_turnconformance_run_multi_turn_tool_calls(): vector_json = r""" { "name": "multi_turn_tool_calls", @@ -5111,7 +5589,7 @@ async def test_vector_83_turnconformance_run_multi_turn_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_84_turnconformance_run_multiple_tool_calls_single_turn(): +async def test_vector_99_turnconformance_run_multiple_tool_calls_single_turn(): vector_json = r""" { "name": "multiple_tool_calls_single_turn", @@ -5292,7 +5770,7 @@ async def test_vector_84_turnconformance_run_multiple_tool_calls_single_turn(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_85_turnconformance_run_no_tool_calls(): +async def test_vector_100_turnconformance_run_no_tool_calls(): vector_json = r""" { "name": "no_tool_calls", @@ -5360,7 +5838,7 @@ async def test_vector_85_turnconformance_run_no_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_86_turnconformance_run_parallel_tools_basic(): +async def test_vector_101_turnconformance_run_parallel_tools_basic(): vector_json = r""" { "name": "parallel_tools_basic", @@ -5533,7 +6011,7 @@ async def test_vector_86_turnconformance_run_parallel_tools_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_87_turnconformance_run_parallel_tools_with_guardrail_deny(): +async def test_vector_102_turnconformance_run_parallel_tools_with_guardrail_deny(): vector_json = r""" { "name": "parallel_tools_with_guardrail_deny", @@ -5724,7 +6202,7 @@ async def test_vector_87_turnconformance_run_parallel_tools_with_guardrail_deny( await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_88_turnconformance_run_parallel_tools_with_tool_error(): +async def test_vector_103_turnconformance_run_parallel_tools_with_tool_error(): vector_json = r""" { "name": "parallel_tools_with_tool_error", @@ -5897,7 +6375,7 @@ async def test_vector_88_turnconformance_run_parallel_tools_with_tool_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_89_turnconformance_run_single_tool_call(): +async def test_vector_104_turnconformance_run_single_tool_call(): vector_json = r""" { "name": "single_tool_call", @@ -6044,7 +6522,7 @@ async def test_vector_89_turnconformance_run_single_tool_call(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_90_turnconformance_run_steering_inject_message(): +async def test_vector_105_turnconformance_run_steering_inject_message(): vector_json = r""" { "name": "steering_inject_message", @@ -6252,7 +6730,7 @@ async def test_vector_90_turnconformance_run_steering_inject_message(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_91_turnconformance_run_steering_multiple_messages(): +async def test_vector_106_turnconformance_run_steering_multiple_messages(): vector_json = r""" { "name": "steering_multiple_messages", @@ -6422,7 +6900,7 @@ async def test_vector_91_turnconformance_run_steering_multiple_messages(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_92_turnconformance_run_tool_not_registered_error(): +async def test_vector_107_turnconformance_run_tool_not_registered_error(): vector_json = r""" { "name": "tool_not_registered_error", @@ -6506,7 +6984,7 @@ async def test_vector_92_turnconformance_run_tool_not_registered_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_93_turnconformance_run_tool_result_message_format(): +async def test_vector_108_turnconformance_run_tool_result_message_format(): vector_json = r""" { "name": "tool_result_message_format", @@ -6627,7 +7105,7 @@ async def test_vector_93_turnconformance_run_tool_result_message_format(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_94_turnconformance_runturn_cancel_before_context(): +async def test_vector_109_turnconformance_runturn_cancel_before_context(): vector_json = r""" { "name": "cancel_before_context", @@ -6659,7 +7137,7 @@ async def test_vector_94_turnconformance_runturn_cancel_before_context(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_95_turnconformance_runturn_delegated_provider_state(): +async def test_vector_110_turnconformance_runturn_delegated_provider_state(): vector_json = r""" { "name": "delegated_provider_state", @@ -6726,7 +7204,7 @@ async def test_vector_95_turnconformance_runturn_delegated_provider_state(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_96_turnconformance_runturn_final_output(): +async def test_vector_111_turnconformance_runturn_final_output(): vector_json = r""" { "name": "final_output", @@ -6771,7 +7249,7 @@ async def test_vector_96_turnconformance_runturn_final_output(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_97_turnconformance_runturn_memory_context_is_model_visible(): +async def test_vector_112_turnconformance_runturn_memory_context_is_model_visible(): vector_json = r""" { "name": "memory_context_is_model_visible", @@ -6833,7 +7311,7 @@ async def test_vector_97_turnconformance_runturn_memory_context_is_model_visible await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_98_turnconformance_runturn_ordered_tool_round(): +async def test_vector_113_turnconformance_runturn_ordered_tool_round(): vector_json = r""" { "name": "ordered_tool_round", @@ -6924,7 +7402,7 @@ async def test_vector_98_turnconformance_runturn_ordered_tool_round(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_99_turnconformance_runturn_permission_denial_is_model_visible(): +async def test_vector_114_turnconformance_runturn_permission_denial_is_model_visible(): vector_json = r""" { "name": "permission_denial_is_model_visible", @@ -6975,7 +7453,7 @@ async def test_vector_99_turnconformance_runturn_permission_denial_is_model_visi await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_100_wireconformance_torequest_anthropic_image_format(): +async def test_vector_115_wireconformance_torequest_anthropic_image_format(): vector_json = r""" { "name": "anthropic_image_format", @@ -7045,7 +7523,7 @@ async def test_vector_100_wireconformance_torequest_anthropic_image_format(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_101_wireconformance_torequest_anthropic_max_tokens_required(): +async def test_vector_116_wireconformance_torequest_anthropic_max_tokens_required(): vector_json = r""" { "name": "anthropic_max_tokens_required", @@ -7100,7 +7578,7 @@ async def test_vector_101_wireconformance_torequest_anthropic_max_tokens_require await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_102_wireconformance_torequest_anthropic_options(): +async def test_vector_117_wireconformance_torequest_anthropic_options(): vector_json = r""" { "name": "anthropic_options", @@ -7169,7 +7647,7 @@ async def test_vector_102_wireconformance_torequest_anthropic_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_103_wireconformance_torequest_anthropic_system_separate(): +async def test_vector_118_wireconformance_torequest_anthropic_system_separate(): vector_json = r""" { "name": "anthropic_system_separate", @@ -7236,7 +7714,7 @@ async def test_vector_103_wireconformance_torequest_anthropic_system_separate(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_104_wireconformance_torequest_anthropic_tool_wire(): +async def test_vector_119_wireconformance_torequest_anthropic_tool_wire(): vector_json = r""" { "name": "anthropic_tool_wire", @@ -7321,7 +7799,7 @@ async def test_vector_104_wireconformance_torequest_anthropic_tool_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_105_wireconformance_torequest_chat_audio_mp3(): +async def test_vector_120_wireconformance_torequest_chat_audio_mp3(): vector_json = r""" { "name": "chat_audio_mp3", @@ -7379,7 +7857,7 @@ async def test_vector_105_wireconformance_torequest_chat_audio_mp3(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_106_wireconformance_torequest_chat_audio_part(): +async def test_vector_121_wireconformance_torequest_chat_audio_part(): vector_json = r""" { "name": "chat_audio_part", @@ -7437,7 +7915,7 @@ async def test_vector_106_wireconformance_torequest_chat_audio_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_107_wireconformance_torequest_chat_image_base64(): +async def test_vector_122_wireconformance_torequest_chat_image_base64(): vector_json = r""" { "name": "chat_image_base64", @@ -7501,7 +7979,7 @@ async def test_vector_107_wireconformance_torequest_chat_image_base64(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_108_wireconformance_torequest_chat_image_part(): +async def test_vector_123_wireconformance_torequest_chat_image_part(): vector_json = r""" { "name": "chat_image_part", @@ -7565,7 +8043,7 @@ async def test_vector_108_wireconformance_torequest_chat_image_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_109_wireconformance_torequest_chat_multipart_content(): +async def test_vector_124_wireconformance_torequest_chat_multipart_content(): vector_json = r""" { "name": "chat_multipart_content", @@ -7629,7 +8107,7 @@ async def test_vector_109_wireconformance_torequest_chat_multipart_content(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_110_wireconformance_torequest_chat_simple(): +async def test_vector_125_wireconformance_torequest_chat_simple(): vector_json = r""" { "name": "chat_simple", @@ -7691,7 +8169,7 @@ async def test_vector_110_wireconformance_torequest_chat_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_111_wireconformance_torequest_chat_single_text_optimized(): +async def test_vector_126_wireconformance_torequest_chat_single_text_optimized(): vector_json = r""" { "name": "chat_single_text_optimized", @@ -7740,7 +8218,7 @@ async def test_vector_111_wireconformance_torequest_chat_single_text_optimized() await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_112_wireconformance_torequest_chat_with_options(): +async def test_vector_127_wireconformance_torequest_chat_with_options(): vector_json = r""" { "name": "chat_with_options", @@ -7808,7 +8286,7 @@ async def test_vector_112_wireconformance_torequest_chat_with_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_113_wireconformance_torequest_embedding_wire(): +async def test_vector_128_wireconformance_torequest_embedding_wire(): vector_json = r""" { "name": "embedding_wire", @@ -7852,7 +8330,7 @@ async def test_vector_113_wireconformance_torequest_embedding_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_114_wireconformance_torequest_image_wire(): +async def test_vector_129_wireconformance_torequest_image_wire(): vector_json = r""" { "name": "image_wire", @@ -7896,7 +8374,7 @@ async def test_vector_114_wireconformance_torequest_image_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_115_wireconformance_torequest_kind_to_json_type_mapping(): +async def test_vector_130_wireconformance_torequest_kind_to_json_type_mapping(): vector_json = r""" { "name": "kind_to_json_type_mapping", @@ -8015,7 +8493,7 @@ async def test_vector_115_wireconformance_torequest_kind_to_json_type_mapping(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_116_wireconformance_torequest_options_additional_properties(): +async def test_vector_131_wireconformance_torequest_options_additional_properties(): vector_json = r""" { "name": "options_additional_properties", @@ -8069,7 +8547,7 @@ async def test_vector_116_wireconformance_torequest_options_additional_propertie await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_117_wireconformance_torequest_options_max_completion_tokens(): +async def test_vector_132_wireconformance_torequest_options_max_completion_tokens(): vector_json = r""" { "name": "options_max_completion_tokens", @@ -8121,7 +8599,7 @@ async def test_vector_117_wireconformance_torequest_options_max_completion_token await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_118_wireconformance_torequest_options_stop_sequences(): +async def test_vector_133_wireconformance_torequest_options_stop_sequences(): vector_json = r""" { "name": "options_stop_sequences", @@ -8179,7 +8657,7 @@ async def test_vector_118_wireconformance_torequest_options_stop_sequences(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_119_wireconformance_torequest_responses_simple(): +async def test_vector_134_wireconformance_torequest_responses_simple(): vector_json = r""" { "name": "responses_simple", @@ -8238,7 +8716,7 @@ async def test_vector_119_wireconformance_torequest_responses_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_120_wireconformance_torequest_responses_structured_output(): +async def test_vector_135_wireconformance_torequest_responses_structured_output(): vector_json = r""" { "name": "responses_structured_output", @@ -8325,7 +8803,7 @@ async def test_vector_120_wireconformance_torequest_responses_structured_output( await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_121_wireconformance_torequest_responses_with_tools(): +async def test_vector_136_wireconformance_torequest_responses_with_tools(): vector_json = r""" { "name": "responses_with_tools", @@ -8405,7 +8883,7 @@ async def test_vector_121_wireconformance_torequest_responses_with_tools(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_122_wireconformance_torequest_structured_output(): +async def test_vector_137_wireconformance_torequest_structured_output(): vector_json = r""" { "name": "structured_output", @@ -8488,7 +8966,7 @@ async def test_vector_122_wireconformance_torequest_structured_output(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_123_wireconformance_torequest_structured_output_nested_optional(): +async def test_vector_138_wireconformance_torequest_structured_output_nested_optional(): vector_json = r""" { "name": "structured_output_nested_optional", @@ -8590,7 +9068,7 @@ async def test_vector_123_wireconformance_torequest_structured_output_nested_opt await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_124_wireconformance_torequest_tools_bindings_stripped(): +async def test_vector_139_wireconformance_torequest_tools_bindings_stripped(): vector_json = r""" { "name": "tools_bindings_stripped", @@ -8682,7 +9160,7 @@ async def test_vector_124_wireconformance_torequest_tools_bindings_stripped(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_125_wireconformance_torequest_tools_function_wire(): +async def test_vector_140_wireconformance_torequest_tools_function_wire(): vector_json = r""" { "name": "tools_function_wire", @@ -8764,7 +9242,7 @@ async def test_vector_125_wireconformance_torequest_tools_function_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_126_wireconformance_torequest_tools_null_when_empty(): +async def test_vector_141_wireconformance_torequest_tools_null_when_empty(): vector_json = r""" { "name": "tools_null_when_empty", @@ -8813,7 +9291,7 @@ async def test_vector_126_wireconformance_torequest_tools_null_when_empty(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_127_wireconformance_torequest_tools_strict_mode(): +async def test_vector_142_wireconformance_torequest_tools_strict_mode(): vector_json = r""" { "name": "tools_strict_mode", diff --git a/runtime/rust/prompty/src/model/vector_conformance.rs b/runtime/rust/prompty/src/model/vector_conformance.rs index 1a6e2ab16..f33b82bbd 100644 --- a/runtime/rust/prompty/src/model/vector_conformance.rs +++ b/runtime/rust/prompty/src/model/vector_conformance.rs @@ -1355,4 +1355,316 @@ pub async fn run_load_conformance_conformance(_ seam: S) async t } // vector: connection_types_load do { - let inputData = "{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}}".data(using: .utf8)! + let inputData = "{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"trustedEndpoints\":[\"https://localhost:8080\"]}".data(using: .utf8)! guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { throw SeamConformanceError("connection_types_load: failed to parse embedded vector input") } @@ -318,7 +318,7 @@ public func runLoadConformanceConformance(_ seam: S) async t } // vector: env_default do { - let inputData = "{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}}".data(using: .utf8)! + let inputData = "{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\",\"apiKey\":\"literal-key\"}}},\"env\":{}}".data(using: .utf8)! guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { throw SeamConformanceError("env_default: failed to parse embedded vector input") } @@ -670,4 +670,241 @@ public func runLoadConformanceConformance(_ seam: S) async t throw SeamConformanceError("tools_openapi_load misrouted") } } + // vector: trusted_endpoint_builtin_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-builtin\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_builtin_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_builtin_allowed misrouted") + } + } + // vector: trusted_endpoint_env_endpoint_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_env_endpoint_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_env_endpoint_allowed misrouted") + } + } + // vector: trusted_endpoint_env_list_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_env_list_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_env_list_allowed misrouted") + } + } + // vector: trusted_endpoint_literal_key_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_literal_key_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_literal_key_allowed misrouted") + } + } + // vector: trusted_endpoint_option_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_option_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_option_allowed misrouted") + } + } + // vector: trusted_endpoint_reference_skipped + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_reference_skipped: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_reference_skipped misrouted") + } + } + // vector: untrusted_endpoint_ambient_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_ambient_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_ambient_rejected: expected an error") + } + } + // vector: untrusted_endpoint_env_default_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_env_default_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_env_default_rejected: expected an error") + } + } + // vector: untrusted_endpoint_env_key_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_env_key_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_env_key_rejected: expected an error") + } + } + // vector: untrusted_endpoint_file_connection_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_file_connection_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_file_connection_rejected: expected an error") + } + } + // vector: untrusted_endpoint_key_with_client_secret_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_key_with_client_secret_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_key_with_client_secret_rejected: expected an error") + } + } + // vector: untrusted_endpoint_mismatched_credential_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_mismatched_credential_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_mismatched_credential_rejected: expected an error") + } + } + // vector: untrusted_endpoint_no_key_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_no_key_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_no_key_rejected: expected an error") + } + } + // vector: untrusted_endpoint_suffix_spoof_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_suffix_spoof_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_suffix_spoof_rejected: expected an error") + } + } + // vector: untrusted_endpoint_wildcard_apex_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_wildcard_apex_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_wildcard_apex_rejected: expected an error") + } + } } diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift index b5cc59dc1..3e00f1af8 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift @@ -245,7 +245,7 @@ final class VectorConformanceTests: XCTestCase { } func testVector21LoadConformanceLoadConnectionTypesLoad() async throws { - guard let vectorData = "{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}".data(using: .utf8), + guard let vectorData = "{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"trustedEndpoints\":[\"https://localhost:8080\"]},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { XCTFail("failed to parse embedded vector") @@ -275,7 +275,7 @@ final class VectorConformanceTests: XCTestCase { } func testVector24LoadConformanceLoadEnvDefault() async throws { - guard let vectorData = "{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}".data(using: .utf8), + guard let vectorData = "{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\",\"apiKey\":\"literal-key\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { XCTFail("failed to parse embedded vector") @@ -514,7 +514,157 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector48MemoryConformanceOperateClearByCategoryAndAll() async throws { + func testVector48LoadConformanceLoadTrustedEndpointBuiltinAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_builtin_allowed\",\"description\":\"Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-builtin\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_env_endpoint_allowed\",\"description\":\"An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector50LoadConformanceLoadTrustedEndpointEnvListAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_env_list_allowed\",\"description\":\"PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_literal_key_allowed\",\"description\":\"A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector52LoadConformanceLoadTrustedEndpointOptionAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_option_allowed\",\"description\":\"A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector53LoadConformanceLoadTrustedEndpointReferenceSkipped() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_reference_skipped\",\"description\":\"kind: reference connections carry no endpoint and are not subject to the trust check\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector54LoadConformanceLoadUntrustedEndpointAmbientRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_ambient_rejected\",\"description\":\"A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec \u{a7}2.5.1)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector60LoadConformanceLoadUntrustedEndpointNoKeyRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector63MemoryConformanceOperateClearByCategoryAndAll() async throws { guard let vectorData = "{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -524,7 +674,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector49MemoryConformanceOperateFormatCoreAndRecallResults() async throws { + func testVector64MemoryConformanceOperateFormatCoreAndRecallResults() async throws { guard let vectorData = "{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -534,7 +684,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector50MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { + func testVector65MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { guard let vectorData = "{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -544,7 +694,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector51MemoryConformanceOperateRecallRanksByWeightedScore() async throws { + func testVector66MemoryConformanceOperateRecallRanksByWeightedScore() async throws { guard let vectorData = "{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -554,7 +704,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { + func testVector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { guard let vectorData = "{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -564,7 +714,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector53MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { + func testVector68MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { guard let vectorData = "{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -574,7 +724,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { + func testVector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { guard let vectorData = "{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\u{e9}\"]},{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\u{e9}\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -584,7 +734,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector55MemoryConformanceOperateRememberDedupsCoreByTags() async throws { + func testVector70MemoryConformanceOperateRememberDedupsCoreByTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -594,7 +744,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { + func testVector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -604,7 +754,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector57MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { + func testVector72MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { guard let vectorData = "{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -614,7 +764,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector58MemoryConformanceOperateSnapshotPortRoundTrip() async throws { + func testVector73MemoryConformanceOperateSnapshotPortRoundTrip() async throws { guard let vectorData = "{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -624,7 +774,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector59MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { + func testVector74MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { guard let vectorData = "{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -634,7 +784,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector60TurnConformanceReplayMaxIterations() async throws { + func testVector75TurnConformanceReplayMaxIterations() async throws { guard let vectorData = "{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -644,7 +794,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector61TurnConformanceReplayNoTool() async throws { + func testVector76TurnConformanceReplayNoTool() async throws { guard let vectorData = "{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -654,7 +804,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector62TurnConformanceReplayPermissionDenied() async throws { + func testVector77TurnConformanceReplayPermissionDenied() async throws { guard let vectorData = "{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -664,7 +814,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector63TurnConformanceReplayToolFailure() async throws { + func testVector78TurnConformanceReplayToolFailure() async throws { guard let vectorData = "{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -674,7 +824,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector64TurnConformanceReplayToolSuccess() async throws { + func testVector79TurnConformanceReplayToolSuccess() async throws { guard let vectorData = "{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -684,7 +834,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector65TurnConformanceRunAssistantToolCallsMetadata() async throws { + func testVector80TurnConformanceRunAssistantToolCallsMetadata() async throws { guard let vectorData = "{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -694,7 +844,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector66TurnConformanceRunAsyncToolFunction() async throws { + func testVector81TurnConformanceRunAsyncToolFunction() async throws { guard let vectorData = "{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -704,7 +854,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector67TurnConformanceRunBindingsInjected() async throws { + func testVector82TurnConformanceRunBindingsInjected() async throws { guard let vectorData = "{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -714,7 +864,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector68TurnConformanceRunCancellationBeforeLlm() async throws { + func testVector83TurnConformanceRunCancellationBeforeLlm() async throws { guard let vectorData = "{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -724,7 +874,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector69TurnConformanceRunCancellationBetweenIterations() async throws { + func testVector84TurnConformanceRunCancellationBetweenIterations() async throws { guard let vectorData = "{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -734,7 +884,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector70TurnConformanceRunCancellationBetweenTools() async throws { + func testVector85TurnConformanceRunCancellationBetweenTools() async throws { guard let vectorData = "{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -744,7 +894,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector71TurnConformanceRunContextNoTrimWhenFits() async throws { + func testVector86TurnConformanceRunContextNoTrimWhenFits() async throws { guard let vectorData = "{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -754,7 +904,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector72TurnConformanceRunContextPreservesSystemMessages() async throws { + func testVector87TurnConformanceRunContextPreservesSystemMessages() async throws { guard let vectorData = "{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -764,7 +914,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector73TurnConformanceRunContextTrimBasic() async throws { + func testVector88TurnConformanceRunContextTrimBasic() async throws { guard let vectorData = "{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -774,7 +924,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector74TurnConformanceRunEmptyToolResult() async throws { + func testVector89TurnConformanceRunEmptyToolResult() async throws { guard let vectorData = "{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -784,7 +934,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector75TurnConformanceRunEventsBasicToolLoop() async throws { + func testVector90TurnConformanceRunEventsBasicToolLoop() async throws { guard let vectorData = "{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -794,7 +944,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector76TurnConformanceRunEventsErrorLogged() async throws { + func testVector91TurnConformanceRunEventsErrorLogged() async throws { guard let vectorData = "{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -804,7 +954,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector77TurnConformanceRunEventsNoTools() async throws { + func testVector92TurnConformanceRunEventsNoTools() async throws { guard let vectorData = "{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -814,7 +964,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector78TurnConformanceRunGuardrailAllPass() async throws { + func testVector93TurnConformanceRunGuardrailAllPass() async throws { guard let vectorData = "{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -824,7 +974,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector79TurnConformanceRunGuardrailInputDeny() async throws { + func testVector94TurnConformanceRunGuardrailInputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -834,7 +984,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector80TurnConformanceRunGuardrailOutputDeny() async throws { + func testVector95TurnConformanceRunGuardrailOutputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -844,7 +994,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector81TurnConformanceRunGuardrailToolDeny() async throws { + func testVector96TurnConformanceRunGuardrailToolDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -854,7 +1004,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector82TurnConformanceRunMaxIterationsExceeded() async throws { + func testVector97TurnConformanceRunMaxIterationsExceeded() async throws { guard let vectorData = "{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -864,7 +1014,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector83TurnConformanceRunMultiTurnToolCalls() async throws { + func testVector98TurnConformanceRunMultiTurnToolCalls() async throws { guard let vectorData = "{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -874,7 +1024,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector84TurnConformanceRunMultipleToolCallsSingleTurn() async throws { + func testVector99TurnConformanceRunMultipleToolCallsSingleTurn() async throws { guard let vectorData = "{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -884,7 +1034,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector85TurnConformanceRunNoToolCalls() async throws { + func testVector100TurnConformanceRunNoToolCalls() async throws { guard let vectorData = "{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -894,7 +1044,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector86TurnConformanceRunParallelToolsBasic() async throws { + func testVector101TurnConformanceRunParallelToolsBasic() async throws { guard let vectorData = "{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -904,7 +1054,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector87TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { + func testVector102TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -914,7 +1064,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector88TurnConformanceRunParallelToolsWithToolError() async throws { + func testVector103TurnConformanceRunParallelToolsWithToolError() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\u{a7}13.6 Parallel Tools + tool error \u{2014} 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -924,7 +1074,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector89TurnConformanceRunSingleToolCall() async throws { + func testVector104TurnConformanceRunSingleToolCall() async throws { guard let vectorData = "{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -934,7 +1084,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector90TurnConformanceRunSteeringInjectMessage() async throws { + func testVector105TurnConformanceRunSteeringInjectMessage() async throws { guard let vectorData = "{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -944,7 +1094,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector91TurnConformanceRunSteeringMultipleMessages() async throws { + func testVector106TurnConformanceRunSteeringMultipleMessages() async throws { guard let vectorData = "{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -954,7 +1104,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector92TurnConformanceRunToolNotRegisteredError() async throws { + func testVector107TurnConformanceRunToolNotRegisteredError() async throws { guard let vectorData = "{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -964,7 +1114,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector93TurnConformanceRunToolResultMessageFormat() async throws { + func testVector108TurnConformanceRunToolResultMessageFormat() async throws { guard let vectorData = "{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -974,7 +1124,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector94TurnConformanceRunTurnCancelBeforeContext() async throws { + func testVector109TurnConformanceRunTurnCancelBeforeContext() async throws { guard let vectorData = "{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -984,7 +1134,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector95TurnConformanceRunTurnDelegatedProviderState() async throws { + func testVector110TurnConformanceRunTurnDelegatedProviderState() async throws { guard let vectorData = "{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -994,7 +1144,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector96TurnConformanceRunTurnFinalOutput() async throws { + func testVector111TurnConformanceRunTurnFinalOutput() async throws { guard let vectorData = "{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1004,7 +1154,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector97TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { + func testVector112TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { guard let vectorData = "{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1014,7 +1164,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector98TurnConformanceRunTurnOrderedToolRound() async throws { + func testVector113TurnConformanceRunTurnOrderedToolRound() async throws { guard let vectorData = "{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1024,7 +1174,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector99TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { + func testVector114TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { guard let vectorData = "{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1034,7 +1184,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector100WireConformanceToRequestAnthropicImageFormat() async throws { + func testVector115WireConformanceToRequestAnthropicImageFormat() async throws { guard let vectorData = "{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1044,7 +1194,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector101WireConformanceToRequestAnthropicMaxTokensRequired() async throws { + func testVector116WireConformanceToRequestAnthropicMaxTokensRequired() async throws { guard let vectorData = "{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1054,7 +1204,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector102WireConformanceToRequestAnthropicOptions() async throws { + func testVector117WireConformanceToRequestAnthropicOptions() async throws { guard let vectorData = "{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1064,7 +1214,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector103WireConformanceToRequestAnthropicSystemSeparate() async throws { + func testVector118WireConformanceToRequestAnthropicSystemSeparate() async throws { guard let vectorData = "{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1074,7 +1224,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector104WireConformanceToRequestAnthropicToolWire() async throws { + func testVector119WireConformanceToRequestAnthropicToolWire() async throws { guard let vectorData = "{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1084,7 +1234,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector105WireConformanceToRequestChatAudioMp3() async throws { + func testVector120WireConformanceToRequestChatAudioMp3() async throws { guard let vectorData = "{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1094,7 +1244,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector106WireConformanceToRequestChatAudioPart() async throws { + func testVector121WireConformanceToRequestChatAudioPart() async throws { guard let vectorData = "{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1104,7 +1254,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector107WireConformanceToRequestChatImageBase64() async throws { + func testVector122WireConformanceToRequestChatImageBase64() async throws { guard let vectorData = "{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1114,7 +1264,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector108WireConformanceToRequestChatImagePart() async throws { + func testVector123WireConformanceToRequestChatImagePart() async throws { guard let vectorData = "{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1124,7 +1274,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector109WireConformanceToRequestChatMultipartContent() async throws { + func testVector124WireConformanceToRequestChatMultipartContent() async throws { guard let vectorData = "{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1134,7 +1284,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector110WireConformanceToRequestChatSimple() async throws { + func testVector125WireConformanceToRequestChatSimple() async throws { guard let vectorData = "{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1144,7 +1294,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector111WireConformanceToRequestChatSingleTextOptimized() async throws { + func testVector126WireConformanceToRequestChatSingleTextOptimized() async throws { guard let vectorData = "{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1154,7 +1304,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector112WireConformanceToRequestChatWithOptions() async throws { + func testVector127WireConformanceToRequestChatWithOptions() async throws { guard let vectorData = "{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1164,7 +1314,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector113WireConformanceToRequestEmbeddingWire() async throws { + func testVector128WireConformanceToRequestEmbeddingWire() async throws { guard let vectorData = "{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1174,7 +1324,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector114WireConformanceToRequestImageWire() async throws { + func testVector129WireConformanceToRequestImageWire() async throws { guard let vectorData = "{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1184,7 +1334,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector115WireConformanceToRequestKindToJsonTypeMapping() async throws { + func testVector130WireConformanceToRequestKindToJsonTypeMapping() async throws { guard let vectorData = "{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1194,7 +1344,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector116WireConformanceToRequestOptionsAdditionalProperties() async throws { + func testVector131WireConformanceToRequestOptionsAdditionalProperties() async throws { guard let vectorData = "{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1204,7 +1354,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector117WireConformanceToRequestOptionsMaxCompletionTokens() async throws { + func testVector132WireConformanceToRequestOptionsMaxCompletionTokens() async throws { guard let vectorData = "{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1214,7 +1364,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector118WireConformanceToRequestOptionsStopSequences() async throws { + func testVector133WireConformanceToRequestOptionsStopSequences() async throws { guard let vectorData = "{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1224,7 +1374,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector119WireConformanceToRequestResponsesSimple() async throws { + func testVector134WireConformanceToRequestResponsesSimple() async throws { guard let vectorData = "{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1234,7 +1384,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector120WireConformanceToRequestResponsesStructuredOutput() async throws { + func testVector135WireConformanceToRequestResponsesStructuredOutput() async throws { guard let vectorData = "{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1244,7 +1394,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector121WireConformanceToRequestResponsesWithTools() async throws { + func testVector136WireConformanceToRequestResponsesWithTools() async throws { guard let vectorData = "{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1254,7 +1404,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector122WireConformanceToRequestStructuredOutput() async throws { + func testVector137WireConformanceToRequestStructuredOutput() async throws { guard let vectorData = "{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1264,7 +1414,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector123WireConformanceToRequestStructuredOutputNestedOptional() async throws { + func testVector138WireConformanceToRequestStructuredOutputNestedOptional() async throws { guard let vectorData = "{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1274,7 +1424,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector124WireConformanceToRequestToolsBindingsStripped() async throws { + func testVector139WireConformanceToRequestToolsBindingsStripped() async throws { guard let vectorData = "{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1284,7 +1434,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector125WireConformanceToRequestToolsFunctionWire() async throws { + func testVector140WireConformanceToRequestToolsFunctionWire() async throws { guard let vectorData = "{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1294,7 +1444,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector126WireConformanceToRequestToolsNullWhenEmpty() async throws { + func testVector141WireConformanceToRequestToolsNullWhenEmpty() async throws { guard let vectorData = "{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1304,7 +1454,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector127WireConformanceToRequestToolsStrictMode() async throws { + func testVector142WireConformanceToRequestToolsStrictMode() async throws { guard let vectorData = "{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { diff --git a/runtime/typescript/packages/core/src/model/vector-conformance.ts b/runtime/typescript/packages/core/src/model/vector-conformance.ts index b7307b453..e0114591c 100644 --- a/runtime/typescript/packages/core/src/model/vector-conformance.ts +++ b/runtime/typescript/packages/core/src/model/vector-conformance.ts @@ -662,4 +662,223 @@ export async function runLoadConformanceConformance( "tools_openapi_load misrouted", ); } + // vector: trusted_endpoint_builtin_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://myres.services.ai.azure.com/api/projects/p"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_builtin_allowed misrouted", + ); + } + // vector: trusted_endpoint_env_endpoint_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://host-chosen.example.com"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_env_endpoint_allowed misrouted", + ); + } + // vector: trusted_endpoint_env_list_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"key","endpoint":"http://localhost:11434/v1"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_env_list_allowed misrouted", + ); + } + // vector: trusted_endpoint_literal_key_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"key","endpoint":"https://anywhere.example.com"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_literal_key_allowed misrouted", + ); + } + // vector: trusted_endpoint_option_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://proxy.example.com:443/openai/v1"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_option_allowed misrouted", + ); + } + // vector: trusted_endpoint_reference_skipped + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"reference","name":"my-connection"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_reference_skipped misrouted", + ); + } + // vector: untrusted_endpoint_ambient_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_ambient_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_env_default_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_env_default_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_env_key_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_env_key_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_file_connection_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_file_connection_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_key_with_client_secret_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_key_with_client_secret_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_mismatched_credential_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_mismatched_credential_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_no_key_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_no_key_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_suffix_spoof_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_suffix_spoof_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_wildcard_apex_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_wildcard_apex_rejected: expected an error", + ); + } } diff --git a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts index af3529740..256c997bc 100644 --- a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts +++ b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts @@ -167,7 +167,7 @@ describe("callable vector conformance", () => { }); it("LoadConformance.load:connection_types_load", async () => { const vector = JSON.parse( - '{"name":"connection_types_load","description":"All connection kinds (key, reference, remote, anonymous) load correctly","stage":"load","input":{"frontmatter":{"name":"connection-test","model":{"id":"gpt-4","connection":{"kind":"anonymous","endpoint":"https://localhost:8080"}}}},"expected":{"model":{"id":"gpt-4","connection":{"kind":"anonymous","endpoint":"https://localhost:8080"}}},"operation":"load"}', + '{"name":"connection_types_load","description":"All connection kinds (key, reference, remote, anonymous) load correctly","stage":"load","input":{"frontmatter":{"name":"connection-test","model":{"id":"gpt-4","connection":{"kind":"anonymous","endpoint":"https://localhost:8080"}}},"trustedEndpoints":["https://localhost:8080"]},"expected":{"model":{"id":"gpt-4","connection":{"kind":"anonymous","endpoint":"https://localhost:8080"}}},"operation":"load"}', ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); @@ -185,7 +185,7 @@ describe("callable vector conformance", () => { }); it("LoadConformance.load:env_default", async () => { const vector = JSON.parse( - '{"name":"env_default","description":"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set","stage":"load","input":{"frontmatter":{"name":"env-default-test","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"${env:MISSING_VAR:fallback_value}"}}},"env":{}},"expected":{"model":{"connection":{"kind":"key","endpoint":"fallback_value"}}},"operation":"load"}', + '{"name":"env_default","description":"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set","stage":"load","input":{"frontmatter":{"name":"env-default-test","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"${env:MISSING_VAR:fallback_value}","apiKey":"literal-key"}}},"env":{}},"expected":{"model":{"connection":{"kind":"key","endpoint":"fallback_value"}}},"operation":"load"}', ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); @@ -327,6 +327,96 @@ describe("callable vector conformance", () => { ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:trusted_endpoint_builtin_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_builtin_allowed","description":"Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration","stage":"load","input":{"frontmatter":{"name":"trusted-builtin","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://myres.services.ai.azure.com/api/projects/p"}}}},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://myres.services.ai.azure.com/api/projects/p"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_env_endpoint_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_env_endpoint_allowed","description":"An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials","stage":"load","input":{"frontmatter":{"name":"trusted-env-endpoint","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"${env:VECTOR_FOUNDRY_ENDPOINT}"}}},"env":{"VECTOR_FOUNDRY_ENDPOINT":"https://host-chosen.example.com"}},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://host-chosen.example.com"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_env_list_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_env_list_allowed","description":"PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint","stage":"load","input":{"frontmatter":{"name":"trusted-env-list","model":{"id":"llama3","connection":{"kind":"key","endpoint":"http://localhost:11434/v1"}}},"env":{"PROMPTY_TRUSTED_ENDPOINTS":"https://other.example.com, http://localhost:11434"}},"expected":{"model":{"connection":{"kind":"key","endpoint":"http://localhost:11434/v1"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_literal_key_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_literal_key_allowed","description":"A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk","stage":"load","input":{"frontmatter":{"name":"trusted-literal-key","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"https://anywhere.example.com","apiKey":"file-supplied-key"}}}},"expected":{"model":{"connection":{"kind":"key","endpoint":"https://anywhere.example.com"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_option_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_option_allowed","description":"A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored","stage":"load","input":{"frontmatter":{"name":"trusted-option","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://proxy.example.com:443/openai/v1"}}},"trustedEndpoints":["HTTPS://PROXY.Example.com"]},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://proxy.example.com:443/openai/v1"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_reference_skipped", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_reference_skipped","description":"kind: reference connections carry no endpoint and are not subject to the trust check","stage":"load","input":{"frontmatter":{"name":"trusted-reference","model":{"id":"gpt-4","connection":{"kind":"reference","name":"my-connection"}}}},"expected":{"model":{"connection":{"kind":"reference","name":"my-connection"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_ambient_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_ambient_rejected","description":"A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)","stage":"load","input":{"frontmatter":{"name":"untrusted-ambient","model":{"id":"gpt-4","provider":"foundry","connection":{"kind":"foundry","endpoint":"https://attacker.example.com/api/projects/p"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_env_default_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_env_default_rejected","description":"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-env-default","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"${env:NONEXISTENT:https://attacker.example.com}"}}},"env":{}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_env_key_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_env_key_rejected","description":"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose","stage":"load","input":{"frontmatter":{"name":"untrusted-env-key","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"https://attacker.example.com","apiKey":"${env:VECTOR_HOST_KEY}"}}},"env":{"VECTOR_HOST_KEY":"host-secret"}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_file_connection_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_file_connection_rejected","description":"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-file-conn","model":{"id":"gpt-4","connection":"${file:conn.json}"}},"files":{"conn.json":{"kind":"foundry","endpoint":"https://attacker.example.com"}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_key_with_client_secret_rejected","description":"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-key-client-secret","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"https://attacker.example.com","clientSecret":"junk"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_mismatched_credential_rejected","description":"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied","stage":"load","input":{"frontmatter":{"name":"untrusted-mismatched-credential","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://attacker.example.com","apiKey":"junk"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_no_key_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_no_key_rejected","description":"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-no-key","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"http://127.0.0.1:8080"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_suffix_spoof_rejected","description":"Origin matching is exact: a host that merely starts with a trusted host is not trusted","stage":"load","input":{"frontmatter":{"name":"untrusted-spoof","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://good.example.com.attacker.example.com"}}},"trustedEndpoints":["https://good.example.com"]},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_wildcard_apex_rejected","description":"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme","stage":"load","input":{"frontmatter":{"name":"untrusted-apex","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"http://myres.openai.azure.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("MemoryConformance.operate:clear_by_category_and_all", async () => { const vector = JSON.parse( '{"name":"clear_by_category_and_all","description":"Clear removes only the requested category, or all memories when no category is supplied.","stage":"memory","input":{"operation":"clear","store":{"entries":[{"content":"fact","category":"core"},{"content":"summary","category":"archival"},{"content":"another summary","category":"archival"},{"content":"insight","category":"insight"}]},"category":"archival"},"expected":{"removed":2,"store":{"entries":[{"content":"fact","category":"core"},{"content":"insight","category":"insight"}]}},"operation":"operate"}', diff --git a/schema/model/conformance/vectors/load.tsp b/schema/model/conformance/vectors/load.tsp index 968c2c1d3..7e6bbb825 100644 --- a/schema/model/conformance/vectors/load.tsp +++ b/schema/model/conformance/vectors/load.tsp @@ -146,7 +146,8 @@ const LoadVectors = """ "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "\${env:MISSING_VAR:fallback_value}" + "endpoint": "\${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }, @@ -269,6 +270,356 @@ const LoadVectors = """ "kind": "file_reference" } }, + { + "name": "untrusted_endpoint_ambient_rejected", + "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ambient", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com/api/projects/p" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_env_key_rejected", + "description": "A literal endpoint with an \${env:} apiKey is rejected: the file would route a host secret to a destination it chose", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": "\${env:VECTOR_HOST_KEY}" + } + } + }, + "env": { + "VECTOR_HOST_KEY": "host-secret" + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_no_key_rejected", + "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-no-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "http://127.0.0.1:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_mismatched_credential_rejected", + "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-mismatched-credential", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com", + "apiKey": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_key_with_client_secret_rejected", + "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-key-client-secret", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "clientSecret": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_env_default_rejected", + "description": "An \${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-default", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "\${env:NONEXISTENT:https://attacker.example.com}" + } + } + }, + "env": {} + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_file_connection_rejected", + "description": "A connection loaded via \${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-file-conn", + "model": { + "id": "gpt-4", + "connection": "\${file:conn.json}" + } + }, + "files": { + "conn.json": { + "kind": "foundry", + "endpoint": "https://attacker.example.com" + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_suffix_spoof_rejected", + "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-spoof", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://good.example.com.attacker.example.com" + } + } + }, + "trustedEndpoints": ["https://good.example.com"] + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_wildcard_apex_rejected", + "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-apex", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://myres.openai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "trusted_endpoint_env_endpoint_allowed", + "description": "An \${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-endpoint", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "\${env:VECTOR_FOUNDRY_ENDPOINT}" + } + } + }, + "env": { + "VECTOR_FOUNDRY_ENDPOINT": "https://host-chosen.example.com" + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://host-chosen.example.com" + } + } + } + }, + { + "name": "trusted_endpoint_literal_key_allowed", + "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-literal-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com", + "apiKey": "file-supplied-key" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com" + } + } + } + }, + { + "name": "trusted_endpoint_option_allowed", + "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-option", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "trustedEndpoints": ["HTTPS://PROXY.Example.com"] + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + } + }, + { + "name": "trusted_endpoint_env_list_allowed", + "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-list", + "model": { + "id": "llama3", + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "env": { + "PROMPTY_TRUSTED_ENDPOINTS": "https://other.example.com, http://localhost:11434" + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + } + }, + { + "name": "trusted_endpoint_builtin_allowed", + "description": "Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-builtin", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + { + "name": "trusted_endpoint_reference_skipped", + "description": "kind: reference connections carry no endpoint and are not subject to the trust check", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-reference", + "model": { + "id": "gpt-4", + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, { "name": "tools_function_load", "description": "Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings", @@ -676,7 +1027,8 @@ const LoadVectors = """ "endpoint": "https://localhost:8080" } } - } + }, + "trustedEndpoints": ["https://localhost:8080"] }, "expected": { "model": { diff --git a/schema/tsp-output/.typra-generated/vectors.json b/schema/tsp-output/.typra-generated/vectors.json index 23feeae50..ac9d1e56f 100644 --- a/schema/tsp-output/.typra-generated/vectors.json +++ b/schema/tsp-output/.typra-generated/vectors.json @@ -841,7 +841,10 @@ "endpoint": "https://localhost:8080" } } - } + }, + "trustedEndpoints": [ + "https://localhost:8080" + ] }, "expected": { "model": { @@ -950,7 +953,8 @@ "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "${env:MISSING_VAR:fallback_value}" + "endpoint": "${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }, @@ -1853,6 +1857,540 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_builtin_allowed", + "description": "Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-builtin", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_env_endpoint_allowed", + "description": "An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-endpoint", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:VECTOR_FOUNDRY_ENDPOINT}" + } + } + }, + "env": { + "VECTOR_FOUNDRY_ENDPOINT": "https://host-chosen.example.com" + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://host-chosen.example.com" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_env_list_allowed", + "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-list", + "model": { + "id": "llama3", + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "env": { + "PROMPTY_TRUSTED_ENDPOINTS": "https://other.example.com, http://localhost:11434" + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_literal_key_allowed", + "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-literal-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com", + "apiKey": "file-supplied-key" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_option_allowed", + "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-option", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "trustedEndpoints": [ + "HTTPS://PROXY.Example.com" + ] + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_reference_skipped", + "description": "kind: reference connections carry no endpoint and are not subject to the trust check", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-reference", + "model": { + "id": "gpt-4", + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_ambient_rejected", + "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ambient", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com/api/projects/p" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_env_default_rejected", + "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-default", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:NONEXISTENT:https://attacker.example.com}" + } + } + }, + "env": {} + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_env_key_rejected", + "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": "${env:VECTOR_HOST_KEY}" + } + } + }, + "env": { + "VECTOR_HOST_KEY": "host-secret" + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_file_connection_rejected", + "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-file-conn", + "model": { + "id": "gpt-4", + "connection": "${file:conn.json}" + } + }, + "files": { + "conn.json": { + "kind": "foundry", + "endpoint": "https://attacker.example.com" + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_key_with_client_secret_rejected", + "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-key-client-secret", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "clientSecret": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_mismatched_credential_rejected", + "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-mismatched-credential", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com", + "apiKey": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_no_key_rejected", + "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-no-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "http://127.0.0.1:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_suffix_spoof_rejected", + "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-spoof", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://good.example.com.attacker.example.com" + } + } + }, + "trustedEndpoints": [ + "https://good.example.com" + ] + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_wildcard_apex_rejected", + "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-apex", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://myres.openai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, { "contract": "MemoryConformance", "namespace": "Prompty.Conformance.Seams", diff --git a/spec/spec.md b/spec/spec.md index 2bbffa218..1cb0bb74f 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -285,6 +285,53 @@ Connection types are discriminated by the `kind` field: | `foundry` | `endpoint` | Microsoft Foundry connection | | `oauth` | `endpoint`, `authenticationMode` | OAuth-based authentication | +#### §2.5.1 Endpoint Trust + +A `.prompty` file is untrusted input. Executors MAY attach **ambient credentials** +(credentials the file did not supply — e.g. `DefaultAzureCredential` tokens, or +`AZURE_OPENAI_API_KEY` / `OPENAI_API_KEY` env fallbacks) to `model.connection.endpoint`. +A file MUST NOT be able to direct ambient credentials to a destination it chose. + +Loaders MUST enforce this at load time (§4.2 step 5) on `model.connection`: + +1. Skip the check when `connection` is absent or `kind` is `reference`. +2. The **credential is file-supplied** only when the field the declared `kind` consumes + is a non-empty literal string (not a `${...}` reference): `apiKey` for `kind: key`, + `clientSecret` for `kind: oauth`. Every other case is ambient, including `${env:}` or + `${file:}` credentials, any other `kind` (e.g. `foundry`, `anonymous`, custom), and + credential fields the declared `kind` does not consume (e.g. `apiKey` on `kind: foundry`). +3. Each **destination** (`endpoint`, and `tokenUrl` when present) is **host-controlled** + when it is absent, or is `${env:VAR}` / `${env:VAR:default}` and `VAR` is set. + A literal value, a `${file:}` value (always untrusted), or an `${env:}` default used + because `VAR` is unset is **file-controlled**. +4. If the credential is ambient and any file-controlled destination is not **trusted**, + raise `untrusted_endpoint`. + +When `model` or `model.connection` is itself a `${file:...}` reference, loaders MUST +resolve it (with §2.11 containment) and apply the same classification to its contents. + +**Trusted origins** are the union of: + +| Source | Example | +| --------------------------------------------- | ----------------------------------------- | +| Built-in first-party provider origins | see below | +| Host load option `trustedEndpoints` | `["https://my-proxy.contoso.com"]` | +| Env var `PROMPTY_TRUSTED_ENDPOINTS` (comma-separated) | `http://localhost:11434,https://*.contoso.com` | + +Built-in: `https://api.openai.com`, `https://api.anthropic.com`, +`https://*.openai.azure.com`, `https://*.services.ai.azure.com`, +`https://*.cognitiveservices.azure.com`, `https://login.microsoftonline.com`. + +**Matching**: compare origins only (scheme + host + port). Scheme and host are +case-insensitive; default ports (`http`:80, `https`:443) are normalized; path, query, +and userinfo are ignored. An entry `scheme://*.domain[:port]` matches any strict +subdomain of `domain` (not `domain` itself). The entry `*` trusts every endpoint. +A destination that does not parse as an absolute `http`/`https` URL is untrusted. +`.prompty` frontmatter MUST NOT be able to grant itself trust. + +Tool connections (§2.9) are out of scope today because no executor attaches ambient +credentials to them; any future handler that does MUST apply the same rule. + ### §2.6 ModelOptions | Property | Type | Description | @@ -815,6 +862,9 @@ flush asynchronously. A failing backend MUST NOT cause the pipeline to fail. | `load` | `(path: string) → PromptAgent` | PromptAgent | | `load_async` | `(path: string) → PromptAgent` | PromptAgent | +Both functions MUST accept host load options `allowedFileRoots` (§2.11) and +`trustedEndpoints` (§2.5.1), named idiomatically per language. + Both functions MUST be traced: emit a `load` span with `inputs = {path: }` and `result = {agent_name: , ...}`. @@ -839,7 +889,11 @@ function load(path): RAISE ValueError("Frontmatter must be a YAML mapping") h. parsed["instructions"] ← body i. RETURN parsed - 5. data ← resolve_references(data, base_dir=parent(path)): + 5. check_endpoint_trust(data, base_dir=parent(path), options): + // §2.5.1 — inspects RAW (unresolved) model.connection + IF credential is ambient AND a file-controlled destination is untrusted: + RAISE untrusted_endpoint + 6. data ← resolve_references(data, base_dir=parent(path)): Walk every value in data recursively: For each string value matching ${protocol:content}: protocol ← lowercase text before first ":" @@ -868,7 +922,7 @@ function load(path): REPLACE with read_file_text(file_path) DEFAULT: Leave value unchanged (unknown protocol) - 6. Apply shorthand expansions: + 7. Apply shorthand expansions: a. IF data["model"] is a string: data["model"] ← {id: data["model"]} b. IF data["template"] is missing: @@ -878,13 +932,13 @@ function load(path): d. For each property in data["inputs"] (if dict form): IF value is a scalar (not a dict): Convert to Property(kind: infer_kind(value), default: value) - 7. Inject kind: + 8. Inject kind: data["kind"] ← "prompt" // .prompty files always produce PromptAgents - 8. agent ← load_typed_agent(data) + 9. agent ← load_typed_agent(data) // Use the generated model loader for the target language // This validates types and produces a PromptAgent instance - 9. RETURN agent + 10. RETURN agent ``` ### §4.3 Dotenv Loading @@ -924,6 +978,11 @@ environment. Environment variables take precedence. | `${env:VAR}` where VAR is not set (no default) | `ValueError` | | `${file:path}` where file does not exist | `FileNotFoundError` | | Invalid YAML in frontmatter | `ValueError` (or YAML parse error) | +| Ambient credential to untrusted endpoint (§2.5.1) | `untrusted_endpoint` | + +The `untrusted_endpoint` message SHOULD name the endpoint and suggest the fixes: +use `${env:VAR}` for the endpoint, add its origin to `trustedEndpoints` / +`PROMPTY_TRUSTED_ENDPOINTS`, or use `kind: reference`. --- From 0f860c58bf18d10114974485cc18c2506325454a Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 12:12:16 -0700 Subject: [PATCH 02/15] feat(python): enforce endpoint trust at load time Add core/endpoint_trust.py and a trusted_endpoints option on load()/load_async(); the vector adapter passes trustedEndpoints. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../prompty/prompty/core/endpoint_trust.py | 125 +++++++++++++++ runtime/python/prompty/prompty/core/loader.py | 46 +++++- .../prompty/tests/model/vector_adapters.py | 11 +- .../prompty/tests/test_endpoint_trust.py | 149 ++++++++++++++++++ 4 files changed, 324 insertions(+), 7 deletions(-) create mode 100644 runtime/python/prompty/prompty/core/endpoint_trust.py create mode 100644 runtime/python/prompty/tests/test_endpoint_trust.py diff --git a/runtime/python/prompty/prompty/core/endpoint_trust.py b/runtime/python/prompty/prompty/core/endpoint_trust.py new file mode 100644 index 000000000..f952affb5 --- /dev/null +++ b/runtime/python/prompty/prompty/core/endpoint_trust.py @@ -0,0 +1,125 @@ +"""Endpoint trust — keep ambient credentials on host-approved destinations (spec §2.5.1). + +A ``.prompty`` file is untrusted input. When its connection carries no +file-supplied credential, executors fall back to ambient credentials +(``DefaultAzureCredential``, ``*_API_KEY`` env vars). This module decides +whether the file-chosen destination may receive them. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterable, Mapping +from typing import Any +from urllib.parse import urlsplit + +from .errors import PromptyLoadError + +__all__ = ["BUILTIN_TRUSTED_ENDPOINTS", "TRUSTED_ENDPOINTS_ENV", "check_connection_trust", "is_trusted_endpoint"] + +TRUSTED_ENDPOINTS_ENV = "PROMPTY_TRUSTED_ENDPOINTS" + +BUILTIN_TRUSTED_ENDPOINTS: tuple[str, ...] = ( + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com", +) + +_DEFAULT_PORTS = {"http": 80, "https": 443} +_CREDENTIAL_KEYS: dict[str, str] = {"key": "apiKey", "oauth": "clientSecret"} +_DESTINATION_KEYS = ("endpoint", "tokenUrl") + + +def _origin(url: str) -> tuple[str, str, int] | None: + """Return ``(scheme, host, port)`` for an absolute http(s) URL, else ``None``.""" + try: + parts = urlsplit(url.strip()) + port = parts.port + except ValueError: + return None + scheme = parts.scheme.lower() + host = (parts.hostname or "").lower() + if scheme not in _DEFAULT_PORTS or not host: + return None + return scheme, host, port if port is not None else _DEFAULT_PORTS[scheme] + + +def _trusted_entries(trusted_endpoints: Iterable[str] | None) -> list[str]: + entries = [*BUILTIN_TRUSTED_ENDPOINTS, *(trusted_endpoints or ())] + entries.extend(os.environ.get(TRUSTED_ENDPOINTS_ENV, "").split(",")) + return [e.strip() for e in entries if isinstance(e, str) and e.strip()] + + +def is_trusted_endpoint(url: str, trusted_endpoints: Iterable[str] | None = None) -> bool: + """Return ``True`` if ``url``'s origin matches a built-in, host, or env trusted origin.""" + entries = _trusted_entries(trusted_endpoints) + if "*" in entries: + return True + target = _origin(url) + if target is None: + return False + scheme, host, port = target + for entry in entries: + allowed = _origin(entry) + if allowed is None or allowed[0] != scheme or allowed[2] != port: + continue + pattern = allowed[1] + if pattern.startswith("*."): + if host.endswith(pattern[1:]): + return True + elif pattern == host: + return True + return False + + +def _is_reference(value: str) -> bool: + return value.startswith("${") and value.endswith("}") + + +def _file_supplied_credential(connection: Mapping[str, Any]) -> bool: + # Only the credential field the declared kind actually consumes counts; any other kind is ambient. + kind = connection.get("kind") + key = _CREDENTIAL_KEYS.get(kind) if isinstance(kind, str) else None + if key is None: + return False + value = connection.get(key) + return isinstance(value, str) and bool(value) and not _is_reference(value) + + +def _file_controlled_destination(value: Any) -> str | None: + """Return the effective destination if the file chose it, else ``None``.""" + if value is None: + return None + if not isinstance(value, str): + return str(value) + if _is_reference(value): + protocol, _, rest = value[2:-1].partition(":") + if protocol.lower() == "env": + var_name, _, default = rest.partition(":") + if os.environ.get(var_name) is not None: + return None + return default or None + return value + + +def check_connection_trust(connection: Any, trusted_endpoints: Iterable[str] | None = None) -> None: + """Raise ``untrusted_endpoint`` if ambient credentials could reach a file-chosen, untrusted destination. + + ``connection`` is the RAW (unresolved) ``model.connection`` mapping. + """ + if not isinstance(connection, Mapping) or connection.get("kind") == "reference": + return + if _file_supplied_credential(connection): + return + for key in _DESTINATION_KEYS: + destination = _file_controlled_destination(connection.get(key)) + if destination is not None and not is_trusted_endpoint(destination, trusted_endpoints): + raise PromptyLoadError( + "untrusted_endpoint", + f"Connection {key} '{destination}' is not trusted to receive ambient credentials. " + f"Set the {key} via ${{env:VAR}}, add its origin to trusted_endpoints or " + f"{TRUSTED_ENDPOINTS_ENV}, or use 'kind: reference'.", + ) diff --git a/runtime/python/prompty/prompty/core/loader.py b/runtime/python/prompty/prompty/core/loader.py index 64205eeaa..33588438f 100644 --- a/runtime/python/prompty/prompty/core/loader.py +++ b/runtime/python/prompty/prompty/core/loader.py @@ -16,6 +16,7 @@ import yaml from ..model import Agent, LoadContext, SaveContext +from .endpoint_trust import check_connection_trust from .errors import PromptyLoadError from .utils import load_prompty, load_prompty_async @@ -27,7 +28,12 @@ # --------------------------------------------------------------------------- -def load(path: str | Path, *, allowed_file_roots: Sequence[str | Path] | None = None) -> Agent: +def load( + path: str | Path, + *, + allowed_file_roots: Sequence[str | Path] | None = None, + trusted_endpoints: Sequence[str] | None = None, +) -> Agent: """Load a ``.prompty`` file and return a typed ``Agent``. Parameters @@ -38,6 +44,10 @@ def load(path: str | Path, *, allowed_file_roots: Sequence[str | Path] | None = Additional directories that ``${file:...}`` references may read from. The prompt file's directory is always allowed. References that resolve outside all allowed roots after canonicalization are rejected. + trusted_endpoints: + Additional origins (``scheme://host[:port]`` or ``scheme://*.domain``) that + may receive ambient credentials when the file supplies none. Merged with + built-in provider origins and ``PROMPTY_TRUSTED_ENDPOINTS``. Returns ------- @@ -52,17 +62,22 @@ def load(path: str | Path, *, allowed_file_roots: Sequence[str | Path] | None = data = load_prompty(path) # 2–7 shared pipeline - return _build_agent(data, path, allowed_file_roots=allowed_file_roots) + return _build_agent(data, path, allowed_file_roots=allowed_file_roots, trusted_endpoints=trusted_endpoints) -async def load_async(path: str | Path, *, allowed_file_roots: Sequence[str | Path] | None = None) -> Agent: +async def load_async( + path: str | Path, + *, + allowed_file_roots: Sequence[str | Path] | None = None, + trusted_endpoints: Sequence[str] | None = None, +) -> Agent: """Async variant of :func:`load`.""" path = Path(path).resolve() if not path.exists(): raise FileNotFoundError(f"Prompty file not found: {path}") data = await load_prompty_async(path) - return _build_agent(data, path, allowed_file_roots=allowed_file_roots) + return _build_agent(data, path, allowed_file_roots=allowed_file_roots, trusted_endpoints=trusted_endpoints) def default_save_context(**kwargs: Any) -> SaveContext: @@ -96,6 +111,7 @@ def _build_agent( path: Path, *, allowed_file_roots: Sequence[str | Path] | None = None, + trusted_endpoints: Sequence[str] | None = None, ) -> Agent: """Shared pipeline that transforms raw frontmatter dict into a Agent.""" @@ -116,6 +132,9 @@ def _build_agent( f"Invalid template format: {tmpl!r} (expected an object with format/parser)", ) + # Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. + check_connection_trust(_raw_connection(data, path, allowed_file_roots), trusted_endpoints) + # 2. Load via Agent.load() with pre_process for ${protocol:value} expansion ctx = LoadContext(pre_process=_pre_process(path, allowed_file_roots=allowed_file_roots)) agent = Agent.load(data, ctx) @@ -130,6 +149,25 @@ def _build_agent( return agent +def _raw_connection( + data: dict[str, Any], + path: Path, + allowed_file_roots: Sequence[str | Path] | None, +) -> Any: + """Return the unresolved ``model.connection``, following whole-value ``${file:}`` references.""" + + def follow(value: Any) -> Any: + if isinstance(value, str) and value.startswith("${") and value.endswith("}"): + protocol, _, ref = value[2:-1].partition(":") + if protocol.lower() == "file": + file_path = _resolve_file_reference(path, ref, allowed_file_roots) + return _load_file_content(file_path) if file_path.exists() else None + return value + + model = follow(data.get("model")) + return follow(model.get("connection")) if isinstance(model, dict) else None + + # --------------------------------------------------------------------------- # Reference resolution via pre_process # --------------------------------------------------------------------------- diff --git a/runtime/python/prompty/tests/model/vector_adapters.py b/runtime/python/prompty/tests/model/vector_adapters.py index 9de184a54..b5076e013 100644 --- a/runtime/python/prompty/tests/model/vector_adapters.py +++ b/runtime/python/prompty/tests/model/vector_adapters.py @@ -271,6 +271,11 @@ def _load_invoke(input: dict, context: dict) -> Any: if "NONEXISTENT" in json.dumps(input): old_env.setdefault("NONEXISTENT", os.environ.get("NONEXISTENT")) os.environ.pop("NONEXISTENT", None) + # Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + if "PROMPTY_TRUSTED_ENDPOINTS" not in env_vars: + old_env.setdefault("PROMPTY_TRUSTED_ENDPOINTS", os.environ.get("PROMPTY_TRUSTED_ENDPOINTS")) + os.environ.pop("PROMPTY_TRUSTED_ENDPOINTS", None) + trusted = input.get("trustedEndpoints") try: # --- input-validation vectors --- @@ -282,18 +287,18 @@ def _load_invoke(input: dict, context: dict) -> Any: with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) if "fixture" in input: - agent = load(SPEC_FIXTURES / input["fixture"]) + agent = load(SPEC_FIXTURES / input["fixture"], trusted_endpoints=trusted) elif "frontmatter_raw" in input: p = tmp_path / "vector.prompty" p.write_text(input["frontmatter_raw"], encoding="utf-8") - agent = load(p) + agent = load(p, trusted_endpoints=trusted) else: frontmatter = dict(input["frontmatter"]) sub = tmp_path / input["agent_subdir"] if input.get("agent_subdir") else tmp_path sub.mkdir(parents=True, exist_ok=True) p = sub / "vector.prompty" _write_prompty(p, frontmatter, input.get("files")) - agent = load(p) + agent = load(p, trusted_endpoints=trusted) return _agent_to_canonical(agent.save(default_save_context(use_shorthand=False))) except Exception as exc: # noqa: BLE001 # Error vectors: attach the canonical {kind, [field]} so the harness can diff --git a/runtime/python/prompty/tests/test_endpoint_trust.py b/runtime/python/prompty/tests/test_endpoint_trust.py new file mode 100644 index 000000000..268e11a6a --- /dev/null +++ b/runtime/python/prompty/tests/test_endpoint_trust.py @@ -0,0 +1,149 @@ +"""Tests for endpoint trust (spec §2.5.1).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from prompty import load +from prompty.core.endpoint_trust import is_trusted_endpoint +from prompty.core.errors import PromptyLoadError + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PROMPTY_TRUSTED_ENDPOINTS", raising=False) + + +def _write(tmp_path: Path, connection: str) -> Path: + p = tmp_path / "t.prompty" + p.write_text(f"---\nname: t\nmodel:\n id: gpt-4\n connection:\n{connection}\n---\nHello\n", encoding="utf-8") + return p + + +class TestIsTrustedEndpoint: + @pytest.mark.parametrize( + "url", + [ + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + ], + ) + def test_builtin_trusted(self, url: str): + assert is_trusted_endpoint(url) + + @pytest.mark.parametrize( + "url", + [ + "http://api.openai.com", # scheme mismatch + "https://api.openai.com:8443", # port mismatch + "https://openai.azure.com", # wildcard never matches apex + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", # userinfo ignored; host is evil.com + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "", + ], + ) + def test_untrusted(self, url: str): + assert not is_trusted_endpoint(url) + + def test_host_option_and_wildcard(self): + assert is_trusted_endpoint("http://localhost:11434/v1", ["http://LOCALHOST:11434"]) + assert is_trusted_endpoint("https://x.contoso.com", ["https://*.contoso.com"]) + assert not is_trusted_endpoint("https://contoso.com", ["https://*.contoso.com"]) + + def test_env_list_and_star(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("PROMPTY_TRUSTED_ENDPOINTS", " https://a.example.com , http://localhost:8080") + assert is_trusted_endpoint("http://localhost:8080/x") + assert not is_trusted_endpoint("http://localhost:8081") + monkeypatch.setenv("PROMPTY_TRUSTED_ENDPOINTS", "*") + assert is_trusted_endpoint("http://anything.example") + + +class TestLoaderEnforcement: + def test_ambient_literal_endpoint_rejected(self, tmp_path: Path): + p = _write(tmp_path, " kind: foundry\n endpoint: http://127.0.0.1:9999") + with pytest.raises(PromptyLoadError) as exc: + load(p) + assert exc.value.kind == "untrusted_endpoint" + assert "127.0.0.1" in str(exc.value) + + def test_env_key_literal_endpoint_rejected(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("SOME_KEY", "secret") + p = _write(tmp_path, " kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}") + with pytest.raises(PromptyLoadError, match="not trusted"): + load(p) + + def test_oauth_token_url_checked(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("SECRET", "s") + conn = ( + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token" + ) + with pytest.raises(PromptyLoadError, match="tokenUrl"): + load(_write(tmp_path, conn)) + + def test_mismatched_credential_field_rejected(self, tmp_path: Path): + p = _write(tmp_path, " kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk") + with pytest.raises(PromptyLoadError, match="not trusted"): + load(p) + p = _write(tmp_path, " kind: key\n endpoint: https://evil.example.com\n clientSecret: junk") + with pytest.raises(PromptyLoadError, match="not trusted"): + load(p) + + def test_literal_key_allowed(self, tmp_path: Path): + agent = load(_write(tmp_path, " kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine")) + assert agent.model.connection.endpoint == "http://127.0.0.1:9999" + + def test_env_endpoint_allowed(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("EP", "http://127.0.0.1:9999") + agent = load(_write(tmp_path, " kind: foundry\n endpoint: ${env:EP}")) + assert agent.model.connection.endpoint == "http://127.0.0.1:9999" + + def test_env_default_used_rejected(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("UNSET_EP", raising=False) + with pytest.raises(PromptyLoadError, match="not trusted"): + load(_write(tmp_path, " kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}")) + + def test_trusted_option_allows(self, tmp_path: Path): + p = _write(tmp_path, " kind: foundry\n endpoint: http://127.0.0.1:9999/api") + agent = load(p, trusted_endpoints=["http://127.0.0.1:9999"]) + assert agent.model.connection.endpoint == "http://127.0.0.1:9999/api" + + @pytest.mark.asyncio + async def test_async_honors_option(self, tmp_path: Path): + from prompty import load_async + + p = _write(tmp_path, " kind: foundry\n endpoint: http://127.0.0.1:9999") + with pytest.raises(PromptyLoadError): + await load_async(p) + agent = await load_async(p, trusted_endpoints=["http://127.0.0.1:9999"]) + assert agent.model.connection.endpoint == "http://127.0.0.1:9999" + + def test_file_model_reference_checked(self, tmp_path: Path): + (tmp_path / "model.json").write_text( + json.dumps({"id": "gpt-4", "connection": {"kind": "foundry", "endpoint": "https://evil.example.com"}}), + encoding="utf-8", + ) + p = tmp_path / "t.prompty" + p.write_text("---\nname: t\nmodel: ${file:model.json}\n---\nHi\n", encoding="utf-8") + with pytest.raises(PromptyLoadError, match="not trusted"): + load(p) + + def test_frontmatter_cannot_self_trust(self, tmp_path: Path): + p = tmp_path / "t.prompty" + p.write_text( + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n", + encoding="utf-8", + ) + with pytest.raises(PromptyLoadError): + load(p) From 293be0a9fab3bea6862ba36373ae556d8ffc08f2 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 12:14:13 -0700 Subject: [PATCH 03/15] docs: document endpoint trust Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/python/prompty/README.md | 8 ++++++++ .../content/docs/core-concepts/file-format.mdx | 10 ++++++++++ .../content/docs/specification/file-format.mdx | 17 +++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/runtime/python/prompty/README.md b/runtime/python/prompty/README.md index 8c0317943..4d4e40a7e 100644 --- a/runtime/python/prompty/README.md +++ b/runtime/python/prompty/README.md @@ -294,6 +294,14 @@ input value. `${file:...}` references are scoped to the containing `.prompty` file's directory by default. Pass `allowed_file_roots` to `load()` / `load_async()` only when the host application explicitly trusts additional shared directories. +### Endpoint Trust + +A `.prompty` file can't send ambient credentials (`DefaultAzureCredential` tokens, `OPENAI_API_KEY` and similar env keys) to an endpoint it picked itself. Unless the connection uses a literal `apiKey` (`kind: key`) or `clientSecret` (`kind: oauth`), a literal `endpoint`/`tokenUrl` must be trusted, or `load()` raises `PromptyLoadError("untrusted_endpoint")`. + +- Built-in trusted origins: `api.openai.com`, `api.anthropic.com`, `*.openai.azure.com`, `*.services.ai.azure.com`, `*.cognitiveservices.azure.com`, `login.microsoftonline.com` (all `https`). +- Supply the endpoint from the host: `endpoint: ${env:MY_ENDPOINT}`. +- Or trust more origins: `load(path, trusted_endpoints=["http://localhost:11434"])` or `PROMPTY_TRUSTED_ENDPOINTS=http://localhost:11434,https://*.contoso.com`. + ### Connection Kinds | Kind | When to use | diff --git a/web/src/content/docs/core-concepts/file-format.mdx b/web/src/content/docs/core-concepts/file-format.mdx index ad49ec5dd..ff7916a6e 100644 --- a/web/src/content/docs/core-concepts/file-format.mdx +++ b/web/src/content/docs/core-concepts/file-format.mdx @@ -426,6 +426,16 @@ connection: ${file:shared/azure-connection.json} cannot grant itself broader filesystem access. + + ## Shorthand Syntax Reference Prompty supports compact shorthands across several properties. Every shorthand diff --git a/web/src/content/docs/specification/file-format.mdx b/web/src/content/docs/specification/file-format.mdx index 75189c4c7..60ffc5464 100644 --- a/web/src/content/docs/specification/file-format.mdx +++ b/web/src/content/docs/specification/file-format.mdx @@ -132,6 +132,23 @@ Connection types are discriminated by the `kind` field: | `foundry` | `endpoint` | Microsoft Foundry connection | | `oauth` | `endpoint`, `authenticationMode` | OAuth-based authentication | +### §2.5.1 Endpoint Trust + +A `.prompty` file is untrusted input, so it MUST NOT be able to direct **ambient +credentials** (`DefaultAzureCredential` tokens, env API keys) to a destination it chose. +Unless the connection supplies its own literal credential (`apiKey` on `kind: key`, +`clientSecret` on `kind: oauth`), a file-controlled `endpoint` or `tokenUrl` (a literal, a +`${file:}` value, or a used `${env:VAR:default}` fallback) must match a trusted origin, or +loading fails with `untrusted_endpoint`. `kind: reference` and `${env:VAR}` endpoints are +host-controlled and pass. + +Trusted origins: built-ins (`https://api.openai.com`, `https://api.anthropic.com`, +`https://*.openai.azure.com`, `https://*.services.ai.azure.com`, +`https://*.cognitiveservices.azure.com`, `https://login.microsoftonline.com`), the +`trustedEndpoints` load option, and the `PROMPTY_TRUSTED_ENDPOINTS` env var +(comma-separated; `*` trusts all). Matching compares scheme, host, and port only; +`*.domain` matches strict subdomains. + ## §2.6 ModelOptions | Property | Type | Description | From c54d82b450dca82da299988bdeef5b9ee6dc5305 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 12:18:06 -0700 Subject: [PATCH 04/15] feat(typescript): enforce endpoint trust at load time Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/typescript/CHANGELOG.md | 7 +- runtime/typescript/packages/core/README.md | 20 ++ .../packages/core/src/core/endpoint-trust.ts | 165 ++++++++++++ .../packages/core/src/core/errors.ts | 3 +- .../packages/core/src/core/index.ts | 6 + .../packages/core/src/core/loader.ts | 87 ++++++- runtime/typescript/packages/core/src/index.ts | 3 + .../core/tests/endpoint-trust.test.ts | 241 ++++++++++++++++++ .../core/tests/model/vector-adapters.ts | 20 +- 9 files changed, 535 insertions(+), 17 deletions(-) create mode 100644 runtime/typescript/packages/core/src/core/endpoint-trust.ts create mode 100644 runtime/typescript/packages/core/tests/endpoint-trust.test.ts diff --git a/runtime/typescript/CHANGELOG.md b/runtime/typescript/CHANGELOG.md index 7f231be67..0af4aeb3d 100644 --- a/runtime/typescript/CHANGELOG.md +++ b/runtime/typescript/CHANGELOG.md @@ -2,7 +2,12 @@ All notable changes to the Prompty TypeScript runtime will be documented here. -## [2.0.1] 2026-10-16 +## [Unreleased] + +### Security +- `load()` enforces spec §2.5.1 endpoint trust: a file whose connection uses ambient credentials may not target an untrusted, file-chosen `endpoint`/`tokenUrl` (`untrusted_endpoint` load error). Add origins via the `trustedEndpoints` load option or `PROMPTY_TRUSTED_ENDPOINTS`. + +## [2.0.1] � 2026-10-16 ### Fixed - OpenAI Responses processor now falls back to nested completed message text when top-level `output_text` is missing or empty (#571) diff --git a/runtime/typescript/packages/core/README.md b/runtime/typescript/packages/core/README.md index 4317f4fdb..fcf45d27f 100644 --- a/runtime/typescript/packages/core/README.md +++ b/runtime/typescript/packages/core/README.md @@ -75,6 +75,26 @@ user: Hello, my name is {{name}}. {{question}} ``` +## Endpoint Trust + +A `.prompty` file is untrusted input. When a connection relies on ambient +credentials (`DefaultAzureCredential`, `OPENAI_API_KEY`, …) rather than a +literal key in the file, `load()` refuses a file-chosen `endpoint` / `tokenUrl` +whose origin is not trusted (error kind `untrusted_endpoint`). Endpoints set via +`${env:VAR}` (with `VAR` set) are host-controlled and always allowed. + +Trusted origins are the built-in provider origins (`https://api.openai.com`, +`https://*.openai.azure.com`, …), plus any you add: + +```typescript +const agent = load("./local.prompty", { + trustedEndpoints: ["http://localhost:11434", "https://*.contoso.com"], +}); +``` + +or via `PROMPTY_TRUSTED_ENDPOINTS=http://localhost:11434,https://*.contoso.com`. +Use `*` to trust every endpoint. + ## Tracing ```typescript diff --git a/runtime/typescript/packages/core/src/core/endpoint-trust.ts b/runtime/typescript/packages/core/src/core/endpoint-trust.ts new file mode 100644 index 000000000..0bf2f0efc --- /dev/null +++ b/runtime/typescript/packages/core/src/core/endpoint-trust.ts @@ -0,0 +1,165 @@ +/** + * Endpoint trust — keep ambient credentials on host-approved destinations (spec §2.5.1). + * + * A `.prompty` file is untrusted input. When its connection carries no + * file-supplied credential, executors fall back to ambient credentials + * (`DefaultAzureCredential`, `*_API_KEY` env vars). This module decides + * whether the file-chosen destination may receive them. + * + * @module + */ + +import { PromptyLoadError } from "./errors.js"; + +/** Env var holding additional comma-separated trusted origins. */ +export const TRUSTED_ENDPOINTS_ENV = "PROMPTY_TRUSTED_ENDPOINTS"; + +/** First-party provider origins that are always trusted. */ +export const BUILTIN_TRUSTED_ENDPOINTS: readonly string[] = Object.freeze([ + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com", +]); + +const DEFAULT_PORTS: Record = { http: 80, https: 443 }; +const CREDENTIAL_KEYS: Record = { + key: "apiKey", + oauth: "clientSecret", +}; +const DESTINATION_KEYS = ["endpoint", "tokenUrl"] as const; + +type Origin = [scheme: string, host: string, port: number]; + +/** Return `[scheme, host, port]` for an absolute http(s) URL, else `undefined`. */ +function origin(url: string): Origin | undefined { + const trimmed = url.trim(); + const match = /^([a-zA-Z][a-zA-Z0-9+.-]*):\/\//.exec(trimmed); + if (!match) return undefined; + const scheme = match[1].toLowerCase(); + if (!(scheme in DEFAULT_PORTS)) return undefined; + let parsed: URL; + try { + parsed = new URL(trimmed); + } catch { + return undefined; + } + const host = parsed.hostname.toLowerCase(); + if (!host) return undefined; + const port = parsed.port ? Number(parsed.port) : DEFAULT_PORTS[scheme]; + return [scheme, host, port]; +} + +function trustedEntries(trustedEndpoints?: Iterable): string[] { + const entries: unknown[] = [ + ...BUILTIN_TRUSTED_ENDPOINTS, + ...(trustedEndpoints ?? []), + ...(process.env[TRUSTED_ENDPOINTS_ENV] ?? "").split(","), + ]; + return entries + .filter((e): e is string => typeof e === "string") + .map((e) => e.trim()) + .filter((e) => e.length > 0); +} + +/** + * Return `true` if `url`'s origin matches a built-in, host-supplied, or + * `PROMPTY_TRUSTED_ENDPOINTS` trusted origin. + */ +export function isTrustedEndpoint( + url: string, + trustedEndpoints?: Iterable, +): boolean { + const entries = trustedEntries(trustedEndpoints); + if (entries.includes("*")) return true; + const target = origin(url); + if (!target) return false; + const [scheme, host, port] = target; + for (const entry of entries) { + const allowed = origin(entry); + if (!allowed || allowed[0] !== scheme || allowed[2] !== port) continue; + const pattern = allowed[1]; + if (pattern.startsWith("*.")) { + if (host.endsWith(pattern.slice(1))) return true; + } else if (pattern === host) { + return true; + } + } + return false; +} + +function isReference(value: string): boolean { + return value.startsWith("${") && value.endsWith("}"); +} + +function fileSuppliedCredential(connection: Record): boolean { + // Only the credential field the declared kind actually consumes counts; any other kind is ambient. + const kind = connection["kind"]; + const key = + typeof kind === "string" && + Object.prototype.hasOwnProperty.call(CREDENTIAL_KEYS, kind) + ? CREDENTIAL_KEYS[kind] + : undefined; + if (key === undefined) return false; + const value = connection[key]; + return typeof value === "string" && value.length > 0 && !isReference(value); +} + +/** Return the effective destination if the file chose it, else `undefined`. */ +function fileControlledDestination(value: unknown): string | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value !== "string") return String(value); + if (isReference(value)) { + const inner = value.slice(2, -1); + const colon = inner.indexOf(":"); + const protocol = colon === -1 ? inner : inner.slice(0, colon); + const rest = colon === -1 ? "" : inner.slice(colon + 1); + if (protocol.toLowerCase() === "env") { + const next = rest.indexOf(":"); + const varName = next === -1 ? rest : rest.slice(0, next); + const defaultVal = next === -1 ? "" : rest.slice(next + 1); + if (process.env[varName] !== undefined) return undefined; + return defaultVal || undefined; + } + } + return value; +} + +/** + * Throw `untrusted_endpoint` if ambient credentials could reach a file-chosen, + * untrusted destination. + * + * @param connection - The RAW (unresolved) `model.connection` mapping. + * @param trustedEndpoints - Host-supplied trusted origins. + */ +export function checkConnectionTrust( + connection: unknown, + trustedEndpoints?: Iterable, +): void { + if ( + typeof connection !== "object" || + connection === null || + Array.isArray(connection) + ) { + return; + } + const conn = connection as Record; + if (conn["kind"] === "reference") return; + if (fileSuppliedCredential(conn)) return; + for (const key of DESTINATION_KEYS) { + const destination = fileControlledDestination(conn[key]); + if ( + destination !== undefined && + !isTrustedEndpoint(destination, trustedEndpoints) + ) { + throw new PromptyLoadError( + "untrusted_endpoint", + `Connection ${key} '${destination}' is not trusted to receive ambient credentials. ` + + `Set the ${key} via \${env:VAR}, add its origin to trustedEndpoints or ` + + `${TRUSTED_ENDPOINTS_ENV}, or use 'kind: reference'.`, + ); + } + } +} diff --git a/runtime/typescript/packages/core/src/core/errors.ts b/runtime/typescript/packages/core/src/core/errors.ts index 31ed71547..f954860eb 100644 --- a/runtime/typescript/packages/core/src/core/errors.ts +++ b/runtime/typescript/packages/core/src/core/errors.ts @@ -17,7 +17,8 @@ export type LoadErrorKind = | "file_not_found" | "invalid_frontmatter" | "invalid_template" - | "missing_required_input"; + | "missing_required_input" + | "untrusted_endpoint"; /** * A typed error raised by the loader/pipeline for negative-path load and diff --git a/runtime/typescript/packages/core/src/core/index.ts b/runtime/typescript/packages/core/src/core/index.ts index f0f59a79a..f96856848 100644 --- a/runtime/typescript/packages/core/src/core/index.ts +++ b/runtime/typescript/packages/core/src/core/index.ts @@ -4,6 +4,12 @@ export * from "./registry.js"; export * from "./connections.js"; export { load, defaultSaveContext, type LoadOptions } from "./loader.js"; export { PromptyLoadError, type LoadErrorKind } from "./errors.js"; +export { + isTrustedEndpoint, + checkConnectionTrust, + BUILTIN_TRUSTED_ENDPOINTS, + TRUSTED_ENDPOINTS_ENV, +} from "./endpoint-trust.js"; export { validateInputs, render, diff --git a/runtime/typescript/packages/core/src/core/loader.ts b/runtime/typescript/packages/core/src/core/loader.ts index 1aa40cfd8..802255883 100644 --- a/runtime/typescript/packages/core/src/core/loader.ts +++ b/runtime/typescript/packages/core/src/core/loader.ts @@ -14,6 +14,7 @@ import matter from "gray-matter"; import { LoadContext, SaveContext } from "../model/context.js"; import { Agent } from "../model/agent.js"; import { PromptyLoadError } from "./errors.js"; +import { checkConnectionTrust } from "./endpoint-trust.js"; export interface LoadOptions { /** @@ -21,6 +22,13 @@ export interface LoadOptions { * The prompt file's directory is always allowed. */ allowedFileRoots?: string[]; + /** + * Additional origins trusted to receive ambient credentials (spec §2.5.1), + * on top of the built-in provider origins and `PROMPTY_TRUSTED_ENDPOINTS`. + * Entries are origins such as `http://localhost:11434` or + * `https://*.contoso.com`; `*` trusts every endpoint. + */ + trustedEndpoints?: string[]; } // --------------------------------------------------------------------------- @@ -132,9 +140,16 @@ function buildAgent( ); } + // Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. + const { agentDir, allowedRoots } = fileRoots(filePath, options); + checkConnectionTrust( + rawConnection(frontmatter, agentDir, allowedRoots), + options.trustedEndpoints, + ); + // 2. Load via Prompty.load() with preProcess for ${protocol:value} expansion const ctx = new LoadContext({ - preProcess: makePreProcess(filePath, options) as ( + preProcess: makePreProcess(agentDir, allowedRoots) as ( data: Record, ) => Record, }); @@ -149,6 +164,64 @@ function buildAgent( return agent; } +function fileRoots( + agentFile: string, + options: LoadOptions, +): { agentDir: string; allowedRoots: string[] } { + const agentDir = realpathSync(dirname(agentFile)); + const allowedRoots = [ + agentDir, + ...(options.allowedFileRoots ?? []).map((root) => + canonicalizeExistingPath(resolve(root)), + ), + ]; + return { agentDir, allowedRoots }; +} + +/** + * Return the unresolved `model.connection`, following whole-value `${file:}` + * references (with the same containment as reference resolution). + */ +function rawConnection( + data: Record, + agentDir: string, + allowedRoots: string[], +): unknown { + const follow = (value: unknown, key: string): unknown => { + if ( + typeof value === "string" && + value.startsWith("${") && + value.endsWith("}") + ) { + const inner = value.slice(2, -1); + const colon = inner.indexOf(":"); + if (colon !== -1 && inner.slice(0, colon).toLowerCase() === "file") { + const reference = inner.slice(colon + 1); + const candidate = isAbsolute(reference) + ? resolve(reference) + : resolve(agentDir, reference); + // Missing files are reported by reference resolution; only containment is enforced here. + if (!existsSync(candidate)) { + if (!isWithinAnyRoot(candidate, allowedRoots)) { + resolveFileReference(agentDir, reference, allowedRoots, key); + } + return undefined; + } + return loadFileContent( + resolveFileReference(agentDir, reference, allowedRoots, key), + ); + } + } + return value; + }; + + const model = follow(data["model"], "model"); + if (typeof model !== "object" || model === null || Array.isArray(model)) { + return undefined; + } + return follow((model as Record)["connection"], "connection"); +} + function rejectExecutableFrontmatter(): never { throw new Error("JavaScript frontmatter is not supported in .prompty files"); } @@ -170,17 +243,9 @@ function rejectExecutableFrontmatter(): never { * provide additional allowed roots via `allowedFileRoots`. */ function makePreProcess( - agentFile: string, - options: LoadOptions, + agentDir: string, + allowedRoots: string[], ): (data: unknown) => unknown { - const agentDir = realpathSync(dirname(agentFile)); - const allowedRoots = [ - agentDir, - ...(options.allowedFileRoots ?? []).map((root) => - canonicalizeExistingPath(resolve(root)), - ), - ]; - return (data: unknown): unknown => { if (typeof data !== "object" || data === null || Array.isArray(data)) { return data; diff --git a/runtime/typescript/packages/core/src/index.ts b/runtime/typescript/packages/core/src/index.ts index a761246a5..cbf804329 100644 --- a/runtime/typescript/packages/core/src/index.ts +++ b/runtime/typescript/packages/core/src/index.ts @@ -58,6 +58,9 @@ export { // Loader load, type LoadOptions, + isTrustedEndpoint, + BUILTIN_TRUSTED_ENDPOINTS, + TRUSTED_ENDPOINTS_ENV, // Pipeline functions validateInputs, diff --git a/runtime/typescript/packages/core/tests/endpoint-trust.test.ts b/runtime/typescript/packages/core/tests/endpoint-trust.test.ts new file mode 100644 index 000000000..de5773485 --- /dev/null +++ b/runtime/typescript/packages/core/tests/endpoint-trust.test.ts @@ -0,0 +1,241 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { load } from "../src/core/loader.js"; +import { isTrustedEndpoint } from "../src/core/endpoint-trust.js"; +import { PromptyLoadError } from "../src/core/errors.js"; + +// Endpoint trust (spec §2.5.1). Mirrors runtime/python/prompty/tests/test_endpoint_trust.py. + +const ENV_KEYS = [ + "PROMPTY_TRUSTED_ENDPOINTS", + "SOME_KEY", + "SECRET", + "EP", + "UNSET_EP", +]; +let savedEnv: Record; +let dir: string; + +beforeEach(() => { + savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); + for (const k of ENV_KEYS) delete process.env[k]; + dir = mkdtempSync(join(tmpdir(), "prompty-trust-")); +}); + +afterEach(() => { + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + rmSync(dir, { recursive: true, force: true }); +}); + +function write(connection: string, base: string = dir): string { + const p = join(base, "t.prompty"); + writeFileSync( + p, + `---\nname: t\nmodel:\n id: gpt-4\n connection:\n${connection}\n---\nHello\n`, + "utf-8", + ); + return p; +} + +function loadError(fn: () => unknown): PromptyLoadError { + try { + fn(); + } catch (err) { + expect(err).toBeInstanceOf(PromptyLoadError); + return err as PromptyLoadError; + } + throw new Error("expected load to throw"); +} + +describe("isTrustedEndpoint", () => { + it.each([ + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + ])("trusts built-in %s", (url) => { + expect(isTrustedEndpoint(url)).toBe(true); + }); + + it.each([ + "http://api.openai.com", // scheme mismatch + "https://api.openai.com:8443", // port mismatch + "https://openai.azure.com", // wildcard never matches apex + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", // userinfo ignored; host is evil.com + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "https:api.openai.com", // not an absolute URL + "", + ])("rejects %s", (url) => { + expect(isTrustedEndpoint(url)).toBe(false); + }); + + it("honors host option entries and wildcards", () => { + expect( + isTrustedEndpoint("http://localhost:11434/v1", [ + "http://LOCALHOST:11434", + ]), + ).toBe(true); + expect( + isTrustedEndpoint("https://x.contoso.com", ["https://*.contoso.com"]), + ).toBe(true); + expect( + isTrustedEndpoint("https://contoso.com", ["https://*.contoso.com"]), + ).toBe(false); + }); + + it("honors PROMPTY_TRUSTED_ENDPOINTS and '*'", () => { + process.env.PROMPTY_TRUSTED_ENDPOINTS = + " https://a.example.com , http://localhost:8080"; + expect(isTrustedEndpoint("http://localhost:8080/x")).toBe(true); + expect(isTrustedEndpoint("http://localhost:8081")).toBe(false); + process.env.PROMPTY_TRUSTED_ENDPOINTS = "*"; + expect(isTrustedEndpoint("http://anything.example")).toBe(true); + }); +}); + +describe("loader enforcement", () => { + it("rejects an ambient credential with a literal endpoint", () => { + const p = write(" kind: foundry\n endpoint: http://127.0.0.1:9999"); + const err = loadError(() => load(p)); + expect(err.kind).toBe("untrusted_endpoint"); + expect(err.message).toContain("127.0.0.1"); + }); + + it("rejects an env-supplied key with a literal endpoint", () => { + process.env.SOME_KEY = "secret"; + const p = write( + " kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}", + ); + expect(() => load(p)).toThrow(/not trusted/); + }); + + it("checks oauth tokenUrl", () => { + process.env.SECRET = "s"; + const p = write( + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token", + ); + expect(() => load(p)).toThrow(/tokenUrl/); + }); + + it("ignores credential fields the kind does not consume", () => { + let p = write( + " kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk", + ); + expect(() => load(p)).toThrow(/not trusted/); + p = write( + " kind: key\n endpoint: https://evil.example.com\n clientSecret: junk", + ); + expect(() => load(p)).toThrow(/not trusted/); + }); + + it("allows a literal key with any endpoint", () => { + const agent = load( + write( + " kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine", + ), + ); + expect((agent.model?.connection as any).endpoint).toBe( + "http://127.0.0.1:9999", + ); + }); + + it("allows a host-controlled ${env:} endpoint", () => { + process.env.EP = "http://127.0.0.1:9999"; + const agent = load(write(" kind: foundry\n endpoint: ${env:EP}")); + expect((agent.model?.connection as any).endpoint).toBe( + "http://127.0.0.1:9999", + ); + }); + + it("rejects an ${env:} default used because the var is unset", () => { + const p = write( + " kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}", + ); + expect(() => load(p)).toThrow(/not trusted/); + }); + + it("allows origins passed via trustedEndpoints", () => { + const p = write( + " kind: foundry\n endpoint: http://127.0.0.1:9999/api", + ); + const agent = load(p, { trustedEndpoints: ["http://127.0.0.1:9999"] }); + expect((agent.model?.connection as any).endpoint).toBe( + "http://127.0.0.1:9999/api", + ); + }); + + it("skips kind: reference", () => { + const agent = load( + write( + " kind: reference\n name: my-conn\n target: https://evil.example.com", + ), + ); + expect((agent.model?.connection as any).kind).toBe("reference"); + }); + + it("follows a whole-value ${file:} model reference", () => { + writeFileSync( + join(dir, "model.json"), + JSON.stringify({ + id: "gpt-4", + connection: { kind: "foundry", endpoint: "https://evil.example.com" }, + }), + "utf-8", + ); + const p = join(dir, "t.prompty"); + writeFileSync(p, "---\nname: t\nmodel: ${file:model.json}\n---\nHi\n"); + expect(() => load(p)).toThrow(/not trusted/); + }); + + it("follows a whole-value ${file:} connection reference", () => { + writeFileSync( + join(dir, "conn.json"), + JSON.stringify({ kind: "foundry", endpoint: "https://evil.example.com" }), + "utf-8", + ); + const p = join(dir, "t.prompty"); + writeFileSync( + p, + "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:conn.json}\n---\nHi\n", + ); + const err = loadError(() => load(p)); + expect(err.kind).toBe("untrusted_endpoint"); + }); + + it("reports traversal ${file:} as file_reference before trust", () => { + const promptDir = join(dir, "prompts"); + mkdirSync(promptDir); + writeFileSync( + join(dir, "conn.json"), + JSON.stringify({ kind: "foundry", endpoint: "https://evil.example.com" }), + "utf-8", + ); + const p = join(promptDir, "t.prompty"); + writeFileSync( + p, + "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:../conn.json}\n---\nHi\n", + ); + expect(loadError(() => load(p)).kind).toBe("file_reference"); + }); + + it("does not let frontmatter grant itself trust", () => { + const p = join(dir, "t.prompty"); + writeFileSync( + p, + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n", + ); + expect(loadError(() => load(p)).kind).toBe("untrusted_endpoint"); + }); +}); diff --git a/runtime/typescript/packages/core/tests/model/vector-adapters.ts b/runtime/typescript/packages/core/tests/model/vector-adapters.ts index 176eacf0c..a271313f6 100644 --- a/runtime/typescript/packages/core/tests/model/vector-adapters.ts +++ b/runtime/typescript/packages/core/tests/model/vector-adapters.ts @@ -86,7 +86,7 @@ import { type TurnToolCall, type TurnToolResult, } from "../../src/index.js"; -import { defaultSaveContext } from "../../src/core/loader.js"; +import { defaultSaveContext, type LoadOptions } from "../../src/core/loader.js"; import { TurnOptions } from "../../src/model/index.js"; import { enrich, mapModel } from "../../src/model/discovery.js"; import { ModelInfo } from "../../src/model/contracts/models/model-info.js"; @@ -375,6 +375,16 @@ function loadInvoke(input: any, _context: AdapterContext): unknown { oldEnv[k] = process.env[k]; process.env[k] = v; } + // Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + if (!("PROMPTY_TRUSTED_ENDPOINTS" in envVars)) { + if (!("PROMPTY_TRUSTED_ENDPOINTS" in oldEnv)) { + oldEnv.PROMPTY_TRUSTED_ENDPOINTS = process.env.PROMPTY_TRUSTED_ENDPOINTS; + } + delete process.env.PROMPTY_TRUSTED_ENDPOINTS; + } + const loadOptions: LoadOptions = { + trustedEndpoints: input.trustedEndpoints, + }; try { // --- input-validation vectors --- @@ -390,18 +400,20 @@ function loadInvoke(input: any, _context: AdapterContext): unknown { const base = mkdtempSync(join(tmpdir(), "prompty-vec-")); try { if ("fixture" in input) { - return saveCanonical(load(join(SPEC_FIXTURES, input.fixture))); + return saveCanonical( + load(join(SPEC_FIXTURES, input.fixture), loadOptions), + ); } if ("frontmatter_raw" in input) { const p = join(base, "vector.prompty"); writeFileSync(p, input.frontmatter_raw, "utf8"); - return saveCanonical(load(p)); + return saveCanonical(load(p, loadOptions)); } const sub = input.agent_subdir ? join(base, input.agent_subdir) : base; mkdirSync(sub, { recursive: true }); const p = join(sub, "vector.prompty"); writePrompty(p, input.frontmatter, input.files); - return saveCanonical(load(p)); + return saveCanonical(load(p, loadOptions)); } finally { rmSync(base, { recursive: true, force: true }); } From 5119e8344ea3c5eb5c1b08d208f5917e4823bad3 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 12:19:11 -0700 Subject: [PATCH 05/15] feat(go): enforce endpoint trust at load time Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/go/prompty/model/endpoint_trust.go | 217 ++++++++++++++++ .../go/prompty/model/endpoint_trust_test.go | 233 ++++++++++++++++++ runtime/go/prompty/model/loader.go | 41 ++- .../prompty/vectoradapters/vectoradapters.go | 25 +- 4 files changed, 508 insertions(+), 8 deletions(-) create mode 100644 runtime/go/prompty/model/endpoint_trust.go create mode 100644 runtime/go/prompty/model/endpoint_trust_test.go diff --git a/runtime/go/prompty/model/endpoint_trust.go b/runtime/go/prompty/model/endpoint_trust.go new file mode 100644 index 000000000..08e138da7 --- /dev/null +++ b/runtime/go/prompty/model/endpoint_trust.go @@ -0,0 +1,217 @@ +package prompty + +// Endpoint trust — keep ambient credentials on host-approved destinations +// (spec §2.5.1). +// +// A .prompty file is untrusted input. When its connection carries no +// file-supplied credential, executors fall back to ambient credentials +// (DefaultAzureCredential, *_API_KEY env vars). This file decides whether the +// file-chosen destination may receive them. + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" +) + +// TrustedEndpointsEnv names the env var holding extra comma-separated trusted origins. +const TrustedEndpointsEnv = "PROMPTY_TRUSTED_ENDPOINTS" + +// BuiltinTrustedEndpoints are first-party provider origins trusted without host configuration. +var BuiltinTrustedEndpoints = []string{ + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com", +} + +var defaultPorts = map[string]int{"http": 80, "https": 443} + +// credentialKeys maps a connection kind to the credential field it consumes. +var credentialKeys = map[string]string{"key": "apiKey", "oauth": "clientSecret"} + +var destinationKeys = []string{"endpoint", "tokenUrl"} + +type origin struct { + scheme string + host string + port int +} + +// parseOrigin returns the (scheme, host, port) of an absolute http(s) URL. +func parseOrigin(raw string) (origin, bool) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return origin{}, false + } + scheme := strings.ToLower(u.Scheme) + host := strings.ToLower(u.Hostname()) + def, ok := defaultPorts[scheme] + if !ok || host == "" { + return origin{}, false + } + port := def + if p := u.Port(); p != "" { + n, err := strconv.Atoi(p) + if err != nil || n < 0 || n > 65535 { + return origin{}, false + } + port = n + } + return origin{scheme: scheme, host: host, port: port}, true +} + +func trustedEntries(trustedEndpoints []string) []string { + entries := make([]string, 0, len(BuiltinTrustedEndpoints)+len(trustedEndpoints)) + entries = append(entries, BuiltinTrustedEndpoints...) + entries = append(entries, trustedEndpoints...) + entries = append(entries, strings.Split(os.Getenv(TrustedEndpointsEnv), ",")...) + out := entries[:0] + for _, e := range entries { + if t := strings.TrimSpace(e); t != "" { + out = append(out, t) + } + } + return out +} + +// IsTrustedEndpoint reports whether rawURL's origin matches a built-in, host +// option, or PROMPTY_TRUSTED_ENDPOINTS origin. +func IsTrustedEndpoint(rawURL string, trustedEndpoints []string) bool { + entries := trustedEntries(trustedEndpoints) + for _, e := range entries { + if e == "*" { + return true + } + } + target, ok := parseOrigin(rawURL) + if !ok { + return false + } + for _, entry := range entries { + allowed, ok := parseOrigin(entry) + if !ok || allowed.scheme != target.scheme || allowed.port != target.port { + continue + } + if strings.HasPrefix(allowed.host, "*.") { + if strings.HasSuffix(target.host, allowed.host[1:]) { + return true + } + } else if allowed.host == target.host { + return true + } + } + return false +} + +func isReference(value string) bool { + return strings.HasPrefix(value, "${") && strings.HasSuffix(value, "}") +} + +func fileSuppliedCredential(connection map[string]interface{}) bool { + // Only the credential field the declared kind actually consumes counts; any other kind is ambient. + kind, _ := connection["kind"].(string) + key, ok := credentialKeys[kind] + if !ok { + return false + } + value, ok := connection[key].(string) + return ok && value != "" && !isReference(value) +} + +// fileControlledDestination returns the effective destination if the file chose it. +func fileControlledDestination(value interface{}) (string, bool) { + if value == nil { + return "", false + } + str, ok := value.(string) + if !ok { + return fmt.Sprintf("%v", value), true + } + if isReference(str) { + protocol, rest, _ := strings.Cut(str[2:len(str)-1], ":") + if strings.ToLower(protocol) == "env" { + varName, def, _ := strings.Cut(rest, ":") + if _, set := os.LookupEnv(varName); set { + return "", false + } + return def, def != "" + } + } + return str, true +} + +// CheckConnectionTrust returns an untrusted_endpoint *LoadError if ambient +// credentials could reach a file-chosen, untrusted destination. connection is +// the RAW (unresolved) model.connection value. +func CheckConnectionTrust(connection interface{}, trustedEndpoints []string) error { + conn, ok := connection.(map[string]interface{}) + if !ok || conn["kind"] == "reference" { + return nil + } + if fileSuppliedCredential(conn) { + return nil + } + for _, key := range destinationKeys { + destination, controlled := fileControlledDestination(conn[key]) + if controlled && !IsTrustedEndpoint(destination, trustedEndpoints) { + return &LoadError{ + Kind: "untrusted_endpoint", + Message: fmt.Sprintf( + "Connection %s '%s' is not trusted to receive ambient credentials. "+ + "Set the %s via ${env:VAR}, add its origin to TrustedEndpoints or %s, "+ + "or use 'kind: reference'.", + key, destination, key, TrustedEndpointsEnv), + } + } + } + return nil +} + +// CheckEndpointTrust applies spec §2.5.1 to RAW (unresolved) frontmatter. It +// follows whole-value ${file:} references for `model` and `model.connection` +// with the same containment as reference resolution, so it must run before +// ResolveReferences. +func CheckEndpointTrust(data map[string]interface{}, parentDir string, allowedRoots, trustedEndpoints []string) error { + conn, err := rawConnection(data, parentDir, allowedRoots) + if err != nil { + return err + } + return CheckConnectionTrust(conn, trustedEndpoints) +} + +func rawConnection(data map[string]interface{}, parentDir string, allowedRoots []string) (interface{}, error) { + follow := func(value interface{}) (interface{}, error) { + str, ok := value.(string) + if !ok || !isReference(str) { + return value, nil + } + protocol, ref, _ := strings.Cut(str[2:len(str)-1], ":") + if strings.ToLower(protocol) != "file" { + return value, nil + } + full := ref + if !filepath.IsAbs(full) { + full = filepath.Join(parentDir, ref) + } + // A missing file is reported by reference resolution, not here. + if _, err := os.Stat(full); err != nil { + return nil, nil + } + return resolveFile(ref, parentDir, allowedRoots) + } + model, err := follow(data["model"]) + if err != nil { + return nil, err + } + modelMap, ok := model.(map[string]interface{}) + if !ok { + return nil, nil + } + return follow(modelMap["connection"]) +} diff --git a/runtime/go/prompty/model/endpoint_trust_test.go b/runtime/go/prompty/model/endpoint_trust_test.go new file mode 100644 index 000000000..4b8a46302 --- /dev/null +++ b/runtime/go/prompty/model/endpoint_trust_test.go @@ -0,0 +1,233 @@ +package prompty + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeTrustPrompty(t *testing.T, dir, connection string) string { + t.Helper() + p := filepath.Join(dir, "t.prompty") + content := "---\nname: t\nmodel:\n id: gpt-4\n connection:\n" + connection + "\n---\nHello\n" + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func expectUntrusted(t *testing.T, err error, contains string) { + t.Helper() + var le *LoadError + if !errors.As(err, &le) || le.Kind != "untrusted_endpoint" { + t.Fatalf("expected untrusted_endpoint LoadError, got %v", err) + } + if contains != "" && !strings.Contains(le.Message, contains) { + t.Fatalf("expected message to contain %q, got %q", contains, le.Message) + } +} + +func connectionEndpoint(t *testing.T, agent Agent) string { + t.Helper() + saved := agent.Save(NewSaveContext()) + model, _ := saved["model"].(map[string]interface{}) + conn, _ := model["connection"].(map[string]interface{}) + endpoint, _ := conn["endpoint"].(string) + return endpoint +} + +func TestIsTrustedEndpointBuiltins(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + for _, u := range []string{ + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + } { + if !IsTrustedEndpoint(u, nil) { + t.Errorf("expected %q to be trusted", u) + } + } +} + +func TestIsTrustedEndpointUntrusted(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + for _, u := range []string{ + "http://api.openai.com", // scheme mismatch + "https://api.openai.com:8443", // port mismatch + "https://openai.azure.com", // wildcard never matches apex + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", // userinfo ignored; host is evil.com + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "", + } { + if IsTrustedEndpoint(u, nil) { + t.Errorf("expected %q to be untrusted", u) + } + } +} + +func TestIsTrustedEndpointHostOptionAndWildcard(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + if !IsTrustedEndpoint("http://localhost:11434/v1", []string{"http://LOCALHOST:11434"}) { + t.Error("expected host option to trust localhost") + } + if !IsTrustedEndpoint("https://x.contoso.com", []string{"https://*.contoso.com"}) { + t.Error("expected wildcard to trust subdomain") + } + if IsTrustedEndpoint("https://contoso.com", []string{"https://*.contoso.com"}) { + t.Error("wildcard must not match apex") + } +} + +func TestIsTrustedEndpointEnvListAndStar(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, " https://a.example.com , http://localhost:8080") + if !IsTrustedEndpoint("http://localhost:8080/x", nil) { + t.Error("expected env list to trust localhost:8080") + } + if IsTrustedEndpoint("http://localhost:8081", nil) { + t.Error("expected localhost:8081 to be untrusted") + } + t.Setenv(TrustedEndpointsEnv, "*") + if !IsTrustedEndpoint("http://anything.example", nil) { + t.Error("expected * to trust everything") + } +} + +func TestLoadAmbientLiteralEndpointRejected(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + p := writeTrustPrompty(t, t.TempDir(), " kind: foundry\n endpoint: http://127.0.0.1:9999") + _, err := LoadPromptyFile(p) + expectUntrusted(t, err, "127.0.0.1") +} + +func TestLoadEnvKeyLiteralEndpointRejected(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + t.Setenv("SOME_KEY", "secret") + p := writeTrustPrompty(t, t.TempDir(), " kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}") + _, err := LoadPromptyFile(p) + expectUntrusted(t, err, "not trusted") +} + +func TestLoadOAuthTokenURLChecked(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + t.Setenv("SECRET", "s") + conn := " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token" + _, err := LoadPromptyFile(writeTrustPrompty(t, t.TempDir(), conn)) + expectUntrusted(t, err, "tokenUrl") +} + +func TestLoadMismatchedCredentialFieldRejected(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + dir := t.TempDir() + _, err := LoadPromptyFile(writeTrustPrompty(t, dir, " kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk")) + expectUntrusted(t, err, "not trusted") + _, err = LoadPromptyFile(writeTrustPrompty(t, dir, " kind: key\n endpoint: https://evil.example.com\n clientSecret: junk")) + expectUntrusted(t, err, "not trusted") +} + +func TestLoadLiteralKeyAllowed(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + agent, err := LoadPromptyFile(writeTrustPrompty(t, t.TempDir(), " kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine")) + if err != nil { + t.Fatal(err) + } + if got := connectionEndpoint(t, agent); got != "http://127.0.0.1:9999" { + t.Fatalf("endpoint = %q", got) + } +} + +func TestLoadEnvEndpointAllowed(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + t.Setenv("EP", "http://127.0.0.1:9999") + agent, err := LoadPromptyFile(writeTrustPrompty(t, t.TempDir(), " kind: foundry\n endpoint: ${env:EP}")) + if err != nil { + t.Fatal(err) + } + if got := connectionEndpoint(t, agent); got != "http://127.0.0.1:9999" { + t.Fatalf("endpoint = %q", got) + } +} + +func TestLoadEnvDefaultUsedRejected(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + os.Unsetenv("UNSET_EP") + _, err := LoadPromptyFile(writeTrustPrompty(t, t.TempDir(), " kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}")) + expectUntrusted(t, err, "not trusted") +} + +func TestLoadTrustedOptionAllows(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + p := writeTrustPrompty(t, t.TempDir(), " kind: foundry\n endpoint: http://127.0.0.1:9999/api") + if _, err := LoadPromptyFile(p); err == nil { + t.Fatal("expected rejection without option") + } + agent, err := LoadPromptyFileWithOptions(p, LoadOptions{TrustedEndpoints: []string{"http://127.0.0.1:9999"}}) + if err != nil { + t.Fatal(err) + } + if got := connectionEndpoint(t, agent); got != "http://127.0.0.1:9999/api" { + t.Fatalf("endpoint = %q", got) + } +} + +func TestLoadFileModelReferenceChecked(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + dir := t.TempDir() + model, _ := json.Marshal(map[string]interface{}{ + "id": "gpt-4", + "connection": map[string]interface{}{"kind": "foundry", "endpoint": "https://evil.example.com"}, + }) + if err := os.WriteFile(filepath.Join(dir, "model.json"), model, 0o644); err != nil { + t.Fatal(err) + } + p := filepath.Join(dir, "t.prompty") + if err := os.WriteFile(p, []byte("---\nname: t\nmodel: ${file:model.json}\n---\nHi\n"), 0o644); err != nil { + t.Fatal(err) + } + _, err := LoadPromptyFile(p) + expectUntrusted(t, err, "not trusted") +} + +func TestLoadFileTraversalStillFileReference(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + base := t.TempDir() + app := filepath.Join(base, "app") + if err := os.MkdirAll(app, 0o755); err != nil { + t.Fatal(err) + } + secret, _ := json.Marshal(map[string]interface{}{"kind": "foundry", "endpoint": "https://evil.example.com"}) + if err := os.WriteFile(filepath.Join(base, "secret.json"), secret, 0o644); err != nil { + t.Fatal(err) + } + p := filepath.Join(app, "t.prompty") + content := "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:../secret.json}\n---\nHi\n" + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + _, err := LoadPromptyFile(p) + var le *LoadError + if !errors.As(err, &le) || le.Kind != "file_traversal" { + t.Fatalf("expected file_traversal, got %v", err) + } +} + +func TestLoadFrontmatterCannotSelfTrust(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + p := filepath.Join(t.TempDir(), "t.prompty") + content := "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n" + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + _, err := LoadPromptyFile(p) + expectUntrusted(t, err, "") +} diff --git a/runtime/go/prompty/model/loader.go b/runtime/go/prompty/model/loader.go index 22aa6cb29..efa54802a 100644 --- a/runtime/go/prompty/model/loader.go +++ b/runtime/go/prompty/model/loader.go @@ -22,7 +22,7 @@ var frontmatterRe = regexp.MustCompile(`(?s)^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+ // LoadError classifies a load failure so the conformance layer (and callers) // can map it onto the expected error contract. type LoadError struct { - Kind string // env | file_missing | file_traversal | template | frontmatter | not_found | required_input + Kind string // env | file_missing | file_traversal | template | frontmatter | not_found | required_input | untrusted_endpoint Field string Message string } @@ -219,22 +219,59 @@ func BuildAgentFromData(data map[string]interface{}) (Agent, error) { return LoadAgent(data, NewLoadContext()) } +// LoadOptions carries host load options (spec §4.1). Frontmatter can never set these. +type LoadOptions struct { + // AllowedFileRoots are extra directories ${file:} references may read from, + // in addition to the prompt file's directory (spec §2.11). + AllowedFileRoots []string + // TrustedEndpoints are extra origins allowed to receive ambient credentials + // (spec §2.5.1), e.g. "https://my-proxy.contoso.com" or "https://*.contoso.com". + TrustedEndpoints []string +} + // LoadPromptyContent parses .prompty text, resolves references relative to // parentDir, and returns the loaded Agent. func LoadPromptyContent(contents, parentDir string, allowedRoots []string) (Agent, error) { + return loadPromptyContent(contents, parentDir, allowedRoots, nil) +} + +// LoadPromptyContentWithOptions is LoadPromptyContent with host load options. +// opts.AllowedFileRoots are appended to allowedRoots. +func LoadPromptyContentWithOptions(contents, parentDir string, allowedRoots []string, opts LoadOptions) (Agent, error) { + return loadPromptyContent(contents, parentDir, appendRoots(allowedRoots, opts.AllowedFileRoots), opts.TrustedEndpoints) +} + +func loadPromptyContent(contents, parentDir string, allowedRoots, trustedEndpoints []string) (Agent, error) { data, err := ParseFrontmatter(contents) if err != nil { return Agent{}, err } + // Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. + if err := CheckEndpointTrust(data, parentDir, allowedRoots, trustedEndpoints); err != nil { + return Agent{}, err + } if err := ResolveReferences(data, parentDir, allowedRoots); err != nil { return Agent{}, err } return BuildAgentFromData(data) } +func appendRoots(roots, extra []string) []string { + out := append([]string{}, roots...) + for _, r := range extra { + out = append(out, canonicalPath(r)) + } + return out +} + // LoadPromptyFile reads a .prompty file from disk and loads it, resolving // references relative to the file's directory. func LoadPromptyFile(path string) (Agent, error) { + return LoadPromptyFileWithOptions(path, LoadOptions{}) +} + +// LoadPromptyFileWithOptions is LoadPromptyFile with host load options. +func LoadPromptyFileWithOptions(path string, opts LoadOptions) (Agent, error) { if _, err := os.Stat(path); err != nil { return Agent{}, &LoadError{Kind: "not_found", Message: fmt.Sprintf("FileNotFoundError: prompty file not found: '%s'", path)} } @@ -243,7 +280,7 @@ func LoadPromptyFile(path string) (Agent, error) { return Agent{}, &LoadError{Kind: "not_found", Message: fmt.Sprintf("FileNotFoundError: %v", err)} } dir := canonicalPath(filepath.Dir(path)) - return LoadPromptyContent(string(content), dir, []string{dir}) + return LoadPromptyContentWithOptions(string(content), dir, []string{dir}, opts) } // ValidateInputs fills declared input defaults and reports missing required diff --git a/runtime/go/prompty/vectoradapters/vectoradapters.go b/runtime/go/prompty/vectoradapters/vectoradapters.go index c23cad29d..310335bb6 100644 --- a/runtime/go/prompty/vectoradapters/vectoradapters.go +++ b/runtime/go/prompty/vectoradapters/vectoradapters.go @@ -412,6 +412,14 @@ func loadInvoke(input any, ctx Context) (any, error) { restore := applyEnv(typed["env"]) defer restore() + // Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + if envMap, _ := typed["env"].(map[string]any); envMap == nil || envMap[prompty.TrustedEndpointsEnv] == nil { + if prev, ok := os.LookupEnv(prompty.TrustedEndpointsEnv); ok { + os.Unsetenv(prompty.TrustedEndpointsEnv) + defer os.Setenv(prompty.TrustedEndpointsEnv, prev) + } + } + opts := prompty.LoadOptions{TrustedEndpoints: toStringSlice(typed["trustedEndpoints"])} // Input-validation vectors carry both a top-level `inputs` map and a // `frontmatter`; full-load vectors nest inputs inside the frontmatter. _, hasInputs := typed["inputs"] @@ -433,11 +441,11 @@ func loadInvoke(input any, ctx Context) (any, error) { switch { case isNonEmptyString(typed["fixture"]): dir := findSpecFixtures(ctx.BaseDir) - agent, err = prompty.LoadPromptyFile(filepath.Join(dir, typed["fixture"].(string))) + agent, err = prompty.LoadPromptyFileWithOptions(filepath.Join(dir, typed["fixture"].(string)), opts) case isNonEmptyString(typed["frontmatter_raw"]): - agent, err = loadFromRaw(typed["frontmatter_raw"].(string)) + agent, err = loadFromRaw(typed["frontmatter_raw"].(string), opts) case isMapAny(typed["frontmatter"]): - agent, err = materializeAndLoad(typed) + agent, err = materializeAndLoad(typed, opts) default: return nil, fmt.Errorf("") } @@ -473,19 +481,19 @@ func unwrapProperties(data map[string]any) map[string]any { return data } -func loadFromRaw(raw string) (prompty.Agent, error) { +func loadFromRaw(raw string, opts prompty.LoadOptions) (prompty.Agent, error) { tmp, err := os.MkdirTemp("", "prompty-load-") if err != nil { return prompty.Agent{}, err } defer os.RemoveAll(tmp) - return prompty.LoadPromptyContent(strings.ReplaceAll(raw, "\r\n", "\n"), tmp, []string{tmp}) + return prompty.LoadPromptyContentWithOptions(strings.ReplaceAll(raw, "\r\n", "\n"), tmp, []string{tmp}, opts) } // materializeAndLoad writes the vector's files (honoring agent_subdir and // allowing `..` keys so path-traversal vectors can plant a target outside the // allowed root), resolves references against the agent directory, and loads. -func materializeAndLoad(input map[string]any) (prompty.Agent, error) { +func materializeAndLoad(input map[string]any, opts prompty.LoadOptions) (prompty.Agent, error) { tempBase, err := os.MkdirTemp("", "prompty-load-") if err != nil { return prompty.Agent{}, err @@ -516,6 +524,9 @@ func materializeAndLoad(input map[string]any) (prompty.Agent, error) { frontmatter := toMapAny(input["frontmatter"]) roots := []string{canonicalDir(agentDir)} + if err := prompty.CheckEndpointTrust(frontmatter, agentDir, roots, opts.TrustedEndpoints); err != nil { + return prompty.Agent{}, err + } if err := prompty.ResolveReferences(frontmatter, agentDir, roots); err != nil { return prompty.Agent{}, err } @@ -647,6 +658,8 @@ func loadErrorPayload(err error) (map[string]any, bool) { kind = "invalid_template" case "required_input": kind = "missing_required_input" + case "untrusted_endpoint": + kind = "untrusted_endpoint" default: return nil, false } From d009f45677f6a31d7452e9e46902b17a2e92a713 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 12:19:11 -0700 Subject: [PATCH 06/15] feat(java): enforce endpoint trust at load time Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/java/README.md | 15 + .../com/microsoft/prompty/EndpointTrust.java | 272 ++++++++++++++++++ .../com/microsoft/prompty/LoadException.java | 9 + .../com/microsoft/prompty/LoadOptions.java | 25 +- .../java/com/microsoft/prompty/Loader.java | 32 +++ .../com/microsoft/prompty/References.java | 16 ++ .../microsoft/prompty/EndpointTrustTest.java | 257 +++++++++++++++++ .../microsoft/prompty/LoadVectorsTest.java | 21 +- .../prompty/model/VectorAdapters.java | 23 +- 9 files changed, 664 insertions(+), 6 deletions(-) create mode 100644 runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java create mode 100644 runtime/java/prompty/src/test/java/com/microsoft/prompty/EndpointTrustTest.java diff --git a/runtime/java/README.md b/runtime/java/README.md index 2d691db36..91f3c49b8 100644 --- a/runtime/java/README.md +++ b/runtime/java/README.md @@ -63,6 +63,21 @@ cd runtime/java ./gradlew :prompty:test # unit tests for the core module only ``` +## Endpoint trust + +A `.prompty` file is untrusted input. When its `model.connection` carries no +file-supplied credential (a literal `apiKey` for `kind: key`, a literal +`clientSecret` for `kind: oauth`), executors fall back to ambient credentials, +so the loader refuses — with `LoadException.Kind.UNTRUSTED_ENDPOINT` — any +file-chosen `endpoint` / `tokenUrl` whose origin is not trusted (spec §2.5.1). +Built-in provider origins are always trusted; hosts add more through +`LoadOptions.trustedEndpoints` or the comma-separated +`PROMPTY_TRUSTED_ENDPOINTS` variable (`*` trusts everything): + +```java +Agent agent = Loader.load(path, LoadOptions.withTrustedEndpoints("http://localhost:11434")); +``` + ## Live provider tests Tests tagged `live` call real providers and are excluded from the default `test` diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java new file mode 100644 index 000000000..d479f6325 --- /dev/null +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java @@ -0,0 +1,272 @@ +package com.microsoft.prompty; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +/** + * Endpoint trust — keeps ambient credentials on host-approved destinations (spec §2.5.1). + * + *

A {@code .prompty} file is untrusted input. When its connection carries no file-supplied + * credential, executors fall back to ambient credentials ({@code DefaultAzureCredential}, + * {@code *_API_KEY} environment variables). This class decides whether the file-chosen destination + * may receive them. + * + *

Trusted origins are the union of {@link #BUILTIN_TRUSTED_ENDPOINTS}, the host's + * {@link LoadOptions#trustedEndpoints()}, and the comma-separated {@value #TRUSTED_ENDPOINTS_ENV} + * variable (read through {@link Environment}). Frontmatter can never contribute to that set. + */ +public final class EndpointTrust { + + /** Environment variable listing additional trusted origins, comma-separated. */ + public static final String TRUSTED_ENDPOINTS_ENV = "PROMPTY_TRUSTED_ENDPOINTS"; + + /** First-party provider origins trusted without host configuration. */ + public static final List BUILTIN_TRUSTED_ENDPOINTS = + List.of( + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com"); + + private static final Map CREDENTIAL_KEYS = + Map.of("key", "apiKey", "oauth", "clientSecret"); + private static final List DESTINATION_KEYS = List.of("endpoint", "tokenUrl"); + + private EndpointTrust() {} + + /** An origin: lowercased scheme and host plus an explicit (or default) port. */ + record Origin(String scheme, String host, int port) {} + + /** + * Parse {@code url} into its origin, or {@code null} when it is not an absolute http(s) URL. + * + *

Hand-rolled rather than built on {@link java.net.URI}, which rejects hosts the other runtimes + * accept (underscores, for instance) and would make the same file trusted in one runtime and not + * another. The rules follow the reference runtime's {@code urlsplit}-based parsing. + */ + static Origin origin(String url) { + String s = url.strip().replace("\t", "").replace("\r", "").replace("\n", ""); + int colon = s.indexOf(':'); + if (colon <= 0 || !isScheme(s.substring(0, colon))) { + return null; + } + String scheme = s.substring(0, colon).toLowerCase(Locale.ROOT); + int defaultPort; + if ("http".equals(scheme)) { + defaultPort = 80; + } else if ("https".equals(scheme)) { + defaultPort = 443; + } else { + return null; + } + String rest = s.substring(colon + 1); + if (!rest.startsWith("//")) { + return null; + } + rest = rest.substring(2); + int end = rest.length(); + for (char delimiter : new char[] {'/', '?', '#'}) { + int index = rest.indexOf(delimiter); + if (index >= 0 && index < end) { + end = index; + } + } + String netloc = rest.substring(0, end); + boolean open = netloc.indexOf('[') >= 0; + boolean close = netloc.indexOf(']') >= 0; + if (open != close) { + return null; + } + + String hostinfo = netloc.substring(netloc.lastIndexOf('@') + 1); + String host; + String port; + int bracket = hostinfo.indexOf('['); + if (bracket >= 0) { + String bracketed = hostinfo.substring(bracket + 1); + int closing = bracketed.indexOf(']'); + host = closing < 0 ? bracketed : bracketed.substring(0, closing); + String after = closing < 0 ? "" : bracketed.substring(closing + 1); + int portColon = after.indexOf(':'); + port = portColon < 0 ? "" : after.substring(portColon + 1); + } else { + int portColon = hostinfo.indexOf(':'); + host = portColon < 0 ? hostinfo : hostinfo.substring(0, portColon); + port = portColon < 0 ? "" : hostinfo.substring(portColon + 1); + } + host = host.toLowerCase(Locale.ROOT); + if (host.isEmpty()) { + return null; + } + if (port.isEmpty()) { + return new Origin(scheme, host, defaultPort); + } + for (int i = 0; i < port.length(); i++) { + char c = port.charAt(i); + if (c < '0' || c > '9') { + return null; + } + } + int number; + try { + number = Integer.parseInt(port); + } catch (NumberFormatException e) { + return null; + } + if (number > 65535) { + return null; + } + return new Origin(scheme, host, number); + } + + private static boolean isScheme(String candidate) { + if (!Character.isLetter(candidate.charAt(0)) || candidate.charAt(0) > 127) { + return false; + } + for (int i = 0; i < candidate.length(); i++) { + char c = candidate.charAt(i); + boolean ok = + (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || c == '+' + || c == '-' + || c == '.'; + if (!ok) { + return false; + } + } + return true; + } + + private static List trustedEntries(List trustedEndpoints) { + List raw = new ArrayList<>(BUILTIN_TRUSTED_ENDPOINTS); + if (trustedEndpoints != null) { + raw.addAll(trustedEndpoints); + } + for (String part : Environment.lookup(TRUSTED_ENDPOINTS_ENV).orElse("").split(",", -1)) { + raw.add(part); + } + List entries = new ArrayList<>(); + for (String entry : raw) { + if (entry != null && !entry.strip().isEmpty()) { + entries.add(entry.strip()); + } + } + return entries; + } + + /** Whether {@code url}'s origin matches a built-in, host, or environment trusted origin. */ + public static boolean isTrustedEndpoint(String url) { + return isTrustedEndpoint(url, List.of()); + } + + /** + * Whether {@code url}'s origin matches a built-in origin, one of {@code trustedEndpoints}, or an + * entry of {@value #TRUSTED_ENDPOINTS_ENV}. + */ + public static boolean isTrustedEndpoint(String url, List trustedEndpoints) { + List entries = trustedEntries(trustedEndpoints); + if (entries.contains("*")) { + return true; + } + Origin target = url == null ? null : origin(url); + if (target == null) { + return false; + } + for (String entry : entries) { + Origin allowed = origin(entry); + if (allowed == null + || !allowed.scheme().equals(target.scheme()) + || allowed.port() != target.port()) { + continue; + } + String pattern = allowed.host(); + if (pattern.startsWith("*.")) { + if (target.host().endsWith(pattern.substring(1))) { + return true; + } + } else if (pattern.equals(target.host())) { + return true; + } + } + return false; + } + + static boolean isReference(String value) { + return value.startsWith("${") && value.endsWith("}"); + } + + private static boolean fileSuppliedCredential(Map connection) { + // Only the credential field the declared kind actually consumes counts; any other kind is + // ambient. + String key = connection.get("kind") instanceof String kind ? CREDENTIAL_KEYS.get(kind) : null; + if (key == null) { + return false; + } + return connection.get(key) instanceof String value && !value.isEmpty() && !isReference(value); + } + + /** The effective destination if the file chose it, or {@code null} when the host did. */ + private static String fileControlledDestination(Object value) { + if (value == null) { + return null; + } + if (!(value instanceof String s)) { + return String.valueOf(value); + } + if (isReference(s)) { + String inner = s.substring(2, s.length() - 1); + int colon = inner.indexOf(':'); + String protocol = colon < 0 ? inner : inner.substring(0, colon); + if ("env".equals(protocol.toLowerCase(Locale.ROOT))) { + String rest = colon < 0 ? "" : inner.substring(colon + 1); + int defaultColon = rest.indexOf(':'); + String varName = defaultColon < 0 ? rest : rest.substring(0, defaultColon); + String defaultValue = defaultColon < 0 ? "" : rest.substring(defaultColon + 1); + Optional actual = Environment.lookup(varName); + if (actual.isPresent()) { + return null; + } + return defaultValue.isEmpty() ? null : defaultValue; + } + } + return s; + } + + /** + * Throw {@link LoadException.Kind#UNTRUSTED_ENDPOINT} if ambient credentials could reach a + * file-chosen, untrusted destination. + * + * @param connection the RAW (unresolved) {@code model.connection} value + * @param trustedEndpoints host-provided trusted origins; may be {@code null} + */ + public static void checkConnectionTrust(Object connection, List trustedEndpoints) { + if (!(connection instanceof Map map) || "reference".equals(map.get("kind"))) { + return; + } + if (fileSuppliedCredential(map)) { + return; + } + for (String key : DESTINATION_KEYS) { + String destination = fileControlledDestination(map.get(key)); + if (destination != null && !isTrustedEndpoint(destination, trustedEndpoints)) { + throw LoadException.untrustedEndpoint( + "Connection " + + key + + " '" + + destination + + "' is not trusted to receive ambient credentials. Set the " + + key + + " via ${env:VAR}, add its origin to trustedEndpoints or " + + TRUSTED_ENDPOINTS_ENV + + ", or use 'kind: reference'."); + } + } + } +} diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadException.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadException.java index e91556bfe..69da01f20 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadException.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadException.java @@ -27,6 +27,11 @@ public enum Kind { * {@code format} and {@code parser}. */ INVALID_TEMPLATE, + /** + * The connection would send ambient credentials to a destination the file chose and the host + * has not trusted (spec §2.5.1). + */ + UNTRUSTED_ENDPOINT, /** Any other load failure. */ OTHER } @@ -75,6 +80,10 @@ public static LoadException invalidTemplate(String detail) { return new LoadException(Kind.INVALID_TEMPLATE, "Invalid template format: " + detail); } + public static LoadException untrustedEndpoint(String detail) { + return new LoadException(Kind.UNTRUSTED_ENDPOINT, detail); + } + public static LoadException other(String detail) { return new LoadException(Kind.OTHER, "Load error: " + detail); } diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadOptions.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadOptions.java index 9e1a07064..fd345d787 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadOptions.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadOptions.java @@ -8,13 +8,24 @@ * * @param allowedFileRoots additional directories that {@code ${file:...}} references may read from. * The prompt file's own directory is always allowed and need not be listed. + * @param trustedEndpoints additional origins (e.g. {@code "https://my-proxy.contoso.com"} or + * {@code "https://*.contoso.com"}) that may receive ambient credentials when a file chooses the + * connection endpoint (spec §2.5.1). Built-in provider origins and the + * {@code PROMPTY_TRUSTED_ENDPOINTS} variable are always consulted; {@code "*"} trusts every + * endpoint. */ -public record LoadOptions(List allowedFileRoots) { +public record LoadOptions(List allowedFileRoots, List trustedEndpoints) { - private static final LoadOptions DEFAULT = new LoadOptions(List.of()); + private static final LoadOptions DEFAULT = new LoadOptions(List.of(), List.of()); public LoadOptions { allowedFileRoots = List.copyOf(allowedFileRoots); + trustedEndpoints = trustedEndpoints == null ? List.of() : List.copyOf(trustedEndpoints); + } + + /** Options with additional {@code ${file:...}} roots and no extra trusted endpoints. */ + public LoadOptions(List allowedFileRoots) { + this(allowedFileRoots, List.of()); } /** Default options: {@code ${file:...}} is confined to the prompt's own directory tree. */ @@ -26,4 +37,14 @@ public static LoadOptions defaults() { public static LoadOptions withAllowedFileRoots(Path... roots) { return new LoadOptions(List.of(roots)); } + + /** Options trusting the given endpoint origins to receive ambient credentials. */ + public static LoadOptions withTrustedEndpoints(String... origins) { + return new LoadOptions(List.of(), List.of(origins)); + } + + /** A copy of these options with {@code trustedEndpoints} replaced. */ + public LoadOptions trustingEndpoints(List origins) { + return new LoadOptions(allowedFileRoots, origins); + } } diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/Loader.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/Loader.java index 15a015d6f..fcb991451 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/Loader.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/Loader.java @@ -23,6 +23,8 @@ *

  • Normalise authored shorthands — dictionary-form inputs and outputs whose value is a bare * scalar become full properties. *
  • Inject {@code kind: "prompt"}; a {@code .prompty} document always describes a prompt agent. + *
  • Check endpoint trust on the raw {@code model.connection} (spec §2.5.1): ambient credentials + * may only flow to a file-chosen destination whose origin the host trusts. *
  • Resolve {@code ${env:...}} and {@code ${file:...}} references. *
  • Hand the resulting map to the generated model's {@code load}. *
  • Record the source path in {@code metadata.__source_path}. @@ -111,6 +113,11 @@ private static com.microsoft.prompty.model.Agent buildAgent( data.put("kind", "prompt"); Path agentDir = filePath.getParent() == null ? Path.of(".") : filePath.getParent(); + + // Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. + EndpointTrust.checkConnectionTrust( + rawConnection(data, agentDir, options.allowedFileRoots()), options.trustedEndpoints()); + References.resolveReferences(data, agentDir, options.allowedFileRoots()); LoadContext context = makeLoadContext(agentDir, options.allowedFileRoots()); @@ -124,6 +131,31 @@ private static com.microsoft.prompty.model.Agent buildAgent( return agent; } + /** The unresolved {@code model.connection}, following whole-value {@code ${file:}} references. */ + private static Object rawConnection( + Map data, Path agentDir, List allowedFileRoots) { + Object model = followFileReference(data.get("model"), agentDir, allowedFileRoots, "model"); + if (!(model instanceof Map modelMap)) { + return null; + } + return followFileReference(modelMap.get("connection"), agentDir, allowedFileRoots, "connection"); + } + + private static Object followFileReference( + Object value, Path agentDir, List allowedFileRoots, String key) { + if (value instanceof String s && EndpointTrust.isReference(s)) { + String inner = s.substring(2, s.length() - 1); + int colon = inner.indexOf(':'); + String protocol = colon < 0 ? inner : inner.substring(0, colon); + if ("file".equals(protocol.toLowerCase(java.util.Locale.ROOT))) { + String reference = colon < 0 ? "" : inner.substring(colon + 1); + return References.readContainedFile(reference, agentDir, allowedFileRoots, key) + .orElse(null); + } + } + return value; + } + /** * A {@code LoadContext} whose {@code preProcess} resolves references as the model tree is walked. * diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/References.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/References.java index 9a8679fe0..1329ca8d6 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/References.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/References.java @@ -100,6 +100,22 @@ public static Optional resolveSingleRef( return tryResolveString(value, "", agentDir, allowedFileRoots, false); } + /** + * Read a {@code ${file:...}} target under the same containment rules as reference resolution. + * + *

    A target that escapes the allowed roots throws {@link LoadException.Kind#FILE_REFERENCE}; a + * contained target that does not exist yields empty, leaving the error to normal resolution. + */ + static Optional readContainedFile( + String relativePath, Path agentDir, List allowedFileRoots, String key) { + Path requested = Path.of(relativePath); + Path full = requested.isAbsolute() ? requested : agentDir.resolve(requested); + if (!Files.exists(full)) { + return Optional.empty(); + } + return resolveFile(relativePath, agentDir, allowedFileRoots, key); + } + private static Optional tryResolveString( String value, String key, Path agentDir, List allowedFileRoots, boolean throwOnError) { if (!value.startsWith("${") || !value.endsWith("}")) { diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/EndpointTrustTest.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/EndpointTrustTest.java new file mode 100644 index 000000000..5eb7dd29e --- /dev/null +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/EndpointTrustTest.java @@ -0,0 +1,257 @@ +package com.microsoft.prompty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.microsoft.prompty.model.Agent; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** Endpoint trust (spec §2.5.1). */ +class EndpointTrustTest { + + private static final String ENV = EndpointTrust.TRUSTED_ENDPOINTS_ENV; + + @TempDir Path tmp; + + @BeforeEach + void maskEnv() { + Environment.mask(ENV); + } + + @AfterEach + void clearEnv() { + Environment.clear(ENV); + } + + private Path write(String connection) throws IOException { + Path p = tmp.resolve("t.prompty"); + Files.writeString( + p, + "---\nname: t\nmodel:\n id: gpt-4\n connection:\n" + connection + "\n---\nHello\n", + StandardCharsets.UTF_8); + return p; + } + + private static LoadException assertUntrusted(Path p, LoadOptions options) { + LoadException e = assertThrows(LoadException.class, () -> Loader.load(p, options)); + assertEquals(LoadException.Kind.UNTRUSTED_ENDPOINT, e.kind()); + return e; + } + + private static LoadException assertUntrusted(Path p) { + return assertUntrusted(p, LoadOptions.defaults()); + } + + private static java.util.Map save(Agent agent) { + return agent.model.connection.save(new com.microsoft.prompty.model.SaveContext()); + } + + @Nested + class IsTrustedEndpoint { + + @ParameterizedTest + @ValueSource( + strings = { + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + }) + void builtinTrusted(String url) { + assertTrue(EndpointTrust.isTrustedEndpoint(url)); + } + + @ParameterizedTest + @ValueSource( + strings = { + "http://api.openai.com", // scheme mismatch + "https://api.openai.com:8443", // port mismatch + "https://openai.azure.com", // wildcard never matches apex + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", // userinfo ignored; host is evil.com + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "", + }) + void untrusted(String url) { + assertFalse(EndpointTrust.isTrustedEndpoint(url)); + } + + @Test + void hostOptionAndWildcard() { + assertTrue( + EndpointTrust.isTrustedEndpoint("http://localhost:11434/v1", List.of("http://LOCALHOST:11434"))); + assertTrue(EndpointTrust.isTrustedEndpoint("https://x.contoso.com", List.of("https://*.contoso.com"))); + assertFalse(EndpointTrust.isTrustedEndpoint("https://contoso.com", List.of("https://*.contoso.com"))); + } + + @Test + void hostWithUnderscoreParses() { + assertTrue( + EndpointTrust.isTrustedEndpoint("https://my_host.example.com/x", List.of("https://my_host.example.com"))); + } + + @Test + void envListAndStar() { + Environment.set(ENV, " https://a.example.com , http://localhost:8080"); + assertTrue(EndpointTrust.isTrustedEndpoint("http://localhost:8080/x")); + assertFalse(EndpointTrust.isTrustedEndpoint("http://localhost:8081")); + Environment.set(ENV, "*"); + assertTrue(EndpointTrust.isTrustedEndpoint("http://anything.example")); + } + } + + @Nested + class LoaderEnforcement { + + @Test + void ambientLiteralEndpointRejected() throws IOException { + LoadException e = assertUntrusted(write(" kind: foundry\n endpoint: http://127.0.0.1:9999")); + assertTrue(e.getMessage().contains("127.0.0.1")); + } + + @Test + void envKeyLiteralEndpointRejected() throws IOException { + Environment.set("SOME_KEY", "secret"); + try { + LoadException e = + assertUntrusted( + write(" kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}")); + assertTrue(e.getMessage().contains("not trusted")); + } finally { + Environment.clear("SOME_KEY"); + } + } + + @Test + void oauthTokenUrlChecked() throws IOException { + Environment.set("SECRET", "s"); + try { + LoadException e = + assertUntrusted( + write( + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token")); + assertTrue(e.getMessage().contains("tokenUrl")); + } finally { + Environment.clear("SECRET"); + } + } + + @Test + void mismatchedCredentialFieldRejected() throws IOException { + assertUntrusted(write(" kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk")); + assertUntrusted(write(" kind: key\n endpoint: https://evil.example.com\n clientSecret: junk")); + } + + @Test + void literalKeyAllowed() throws IOException { + Agent agent = Loader.load(write(" kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine")); + assertEquals( + "http://127.0.0.1:9999", + ((com.microsoft.prompty.model.ApiKeyConnection) agent.model.connection).endpoint); + } + + @Test + void envEndpointAllowed() throws IOException { + Environment.set("EP", "http://127.0.0.1:9999"); + try { + Agent agent = Loader.load(write(" kind: foundry\n endpoint: ${env:EP}")); + assertEquals("http://127.0.0.1:9999", save(agent).get("endpoint")); + } finally { + Environment.clear("EP"); + } + } + + @Test + void envDefaultUsedRejected() throws IOException { + Environment.mask("UNSET_EP"); + try { + assertUntrusted(write(" kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}")); + } finally { + Environment.clear("UNSET_EP"); + } + } + + @Test + void trustedOptionAllows() throws IOException { + Path p = write(" kind: foundry\n endpoint: http://127.0.0.1:9999/api"); + Agent agent = Loader.load(p, LoadOptions.withTrustedEndpoints("http://127.0.0.1:9999")); + assertEquals("http://127.0.0.1:9999/api", save(agent).get("endpoint")); + } + + @Test + void loadFromStringHonorsOption() { + String raw = + "---\nname: t\nmodel:\n id: gpt-4\n connection:\n kind: foundry\n" + + " endpoint: http://127.0.0.1:9999\n---\nHi\n"; + Path base = tmp.resolve("virtual.prompty"); + LoadException e = assertThrows(LoadException.class, () -> Loader.loadFromString(raw, base)); + assertEquals(LoadException.Kind.UNTRUSTED_ENDPOINT, e.kind()); + Agent agent = + Loader.loadFromString(raw, base, LoadOptions.withTrustedEndpoints("http://127.0.0.1:9999")); + assertEquals("http://127.0.0.1:9999", save(agent).get("endpoint")); + } + + @Test + void fileModelReferenceChecked() throws IOException { + Files.writeString( + tmp.resolve("model.json"), + "{\"id\": \"gpt-4\", \"connection\": {\"kind\": \"foundry\", \"endpoint\": \"https://evil.example.com\"}}", + StandardCharsets.UTF_8); + Path p = tmp.resolve("t.prompty"); + Files.writeString(p, "---\nname: t\nmodel: ${file:model.json}\n---\nHi\n", StandardCharsets.UTF_8); + LoadException e = assertUntrusted(p); + assertTrue(e.getMessage().contains("not trusted")); + } + + @Test + void traversalFileConnectionRaisesFileReference() throws IOException { + Path sub = Files.createDirectories(tmp.resolve("agent")); + Files.writeString( + tmp.resolve("conn.json"), + "{\"kind\": \"foundry\", \"endpoint\": \"https://evil.example.com\"}", + StandardCharsets.UTF_8); + Path p = sub.resolve("t.prompty"); + Files.writeString( + p, + "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:../conn.json}\n---\nHi\n", + StandardCharsets.UTF_8); + LoadException e = assertThrows(LoadException.class, () -> Loader.load(p)); + assertEquals(LoadException.Kind.FILE_REFERENCE, e.kind()); + } + + @Test + void frontmatterCannotSelfTrust() throws IOException { + Path p = tmp.resolve("t.prompty"); + Files.writeString( + p, + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n", + StandardCharsets.UTF_8); + assertUntrusted(p); + } + + @Test + void referenceKindSkipped() throws IOException { + Agent agent = Loader.load(write(" kind: reference\n name: my-connection")); + assertEquals("reference", save(agent).get("kind")); + } + } +} diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/LoadVectorsTest.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/LoadVectorsTest.java index b16d93937..63df92124 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/LoadVectorsTest.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/LoadVectorsTest.java @@ -142,6 +142,7 @@ private static Map canonicalError(Throwable thrown, Agent agent, case ENV_VAR_NOT_SET -> "env_var_not_set"; case FILE_REFERENCE -> "file_reference"; case INVALID_TEMPLATE -> "invalid_template"; + case UNTRUSTED_ENDPOINT -> "untrusted_endpoint"; case OTHER -> "other"; }); } else if (thrown instanceof InvokerException invoker @@ -184,7 +185,7 @@ private static String firstMissingRequired(Agent agent, Map prov private Agent load(Map input) { String fixture = SpecVectors.string(input, "fixture"); if (fixture != null) { - return Loader.load(SpecVectors.fixtures().resolve(fixture)); + return Loader.load(SpecVectors.fixtures().resolve(fixture), loadOptions(input)); } String raw = SpecVectors.string(input, "frontmatter_raw"); @@ -217,7 +218,7 @@ private Agent load(Map input) { // `${file:}` references resolve relative to the agent's own directory, so the vector's virtual // files have to sit beside a virtual agent path rather than beside the test's working directory. - return Loader.loadFromString(raw, agentDir.resolve("virtual.prompty")); + return Loader.loadFromString(raw, agentDir.resolve("virtual.prompty"), loadOptions(input)); } private static String toYaml(Object frontmatter) { @@ -273,9 +274,25 @@ private static List setEnv(Map input, Map input) { + List trusted = new ArrayList<>(); + if (input.get("trustedEndpoints") instanceof List list) { + for (Object item : list) { + trusted.add(String.valueOf(item)); + } + } + return new LoadOptions(List.of(), trusted); + } + private static void clearEnv(List keys) { for (String key : keys) { Environment.clear(key); diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java index 7b8d29bfb..1f8facb13 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java @@ -2,9 +2,11 @@ import com.microsoft.prompty.CancellationToken; import com.microsoft.prompty.Discovery; +import com.microsoft.prompty.EndpointTrust; import com.microsoft.prompty.Environment; import com.microsoft.prompty.InvokerException; import com.microsoft.prompty.LoadException; +import com.microsoft.prompty.LoadOptions; import com.microsoft.prompty.Loader; import com.microsoft.prompty.Memory; import com.microsoft.prompty.Messages; @@ -319,6 +321,7 @@ private static Map loadKindPayload(LoadException e) { case ENV_VAR_NOT_SET -> "env_var_not_set"; case FILE_REFERENCE -> "file_reference"; case INVALID_TEMPLATE -> "invalid_template"; + case UNTRUSTED_ENDPOINT -> "untrusted_endpoint"; case OTHER -> "other"; }); return payload; @@ -349,7 +352,7 @@ private static String firstMissingRequired(Agent agent, Map prov private static Agent loadAgent(Map input) { String fixture = string(input.get("fixture")); if (!fixture.isEmpty()) { - return Loader.load(SpecVectors.fixtures().resolve(fixture)); + return Loader.load(SpecVectors.fixtures().resolve(fixture), loadOptions(input)); } String raw = input.get("frontmatter_raw") instanceof String r ? r : null; Map files = asMap(input.get("files")); @@ -371,7 +374,7 @@ private static Agent loadAgent(Map input) { Object content = file.getValue(); write(target, content instanceof String text ? text : TypraJson.stringify(content)); } - return Loader.loadFromString(raw, agentDir.resolve("virtual.prompty")); + return Loader.loadFromString(raw, agentDir.resolve("virtual.prompty"), loadOptions(input)); } private static String toYaml(Object frontmatter) { @@ -420,9 +423,25 @@ private static List setEnv(Map input, Map input) { + List trusted = new ArrayList<>(); + if (input.get("trustedEndpoints") instanceof List list) { + for (Object item : list) { + trusted.add(String.valueOf(item)); + } + } + return new LoadOptions(List.of(), trusted); + } + private static void clearEnv(List keys) { for (String key : keys) { Environment.clear(key); From 3f5b60ca8f08b214fe9e7dda52df7aca8c969df6 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 12:22:02 -0700 Subject: [PATCH 07/15] feat(rust): enforce endpoint trust at load time Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/rust/prompty/README.md | 27 + runtime/rust/prompty/src/loader.rs | 26 +- .../rust/prompty/src/loader/endpoint_trust.rs | 538 ++++++++++++++++++ runtime/rust/prompty/src/loader/error.rs | 6 + runtime/rust/prompty/src/loader/resolve.rs | 37 ++ runtime/rust/prompty/tests/loader_test.rs | 15 +- .../prompty/tests/model/vector_adapters.rs | 27 +- 7 files changed, 668 insertions(+), 8 deletions(-) create mode 100644 runtime/rust/prompty/src/loader/endpoint_trust.rs diff --git a/runtime/rust/prompty/README.md b/runtime/rust/prompty/README.md index 602afb338..ae71736af 100644 --- a/runtime/rust/prompty/README.md +++ b/runtime/rust/prompty/README.md @@ -40,6 +40,33 @@ structured output. `TurnOptions::validator` runs on that final payload after any guardrail rewrite and before the durable success commit; rejection records a failed turn. `max_llm_retries` controls model attempts for both simple and tool-calling turns. +### Load options and endpoint trust + +`load_with_options()` / `load_async_with_options()` accept `LoadOptions`: + +- `allowed_file_roots` — extra directories `${file:...}` references may read from. +- `trusted_endpoints` — extra origins allowed to receive ambient credentials. + +A `.prompty` file is untrusted. When its connection has no literal file-supplied +credential (`apiKey` for `kind: key`, `clientSecret` for `kind: oauth`), executors fall +back to ambient credentials (Entra ID tokens, `*_API_KEY` env vars). Loading fails with +`LoadError::UntrustedEndpoint` if such a connection's `endpoint` / `tokenUrl` is chosen +by the file (a literal, `${file:...}`, or an `${env:VAR:default}` default) and its origin +is not trusted. Trusted origins are the built-in provider origins (`https://api.openai.com`, +`https://api.anthropic.com`, `https://*.openai.azure.com`, `https://*.services.ai.azure.com`, +`https://*.cognitiveservices.azure.com`, `https://login.microsoftonline.com`), plus +`trusted_endpoints`, plus the comma-separated `PROMPTY_TRUSTED_ENDPOINTS` env var (`*` +trusts everything). Endpoints supplied via `${env:VAR}` with `VAR` set are host-controlled +and always allowed. + +```rust +let options = prompty::LoadOptions { + trusted_endpoints: vec!["http://localhost:11434".into()], + ..Default::default() +}; +let agent = prompty::load_with_options("local.prompty", options)?; +``` + ## Live turns, durability, and resume `turn()` is the convenient live API. For hosts that own a session journal or need to diff --git a/runtime/rust/prompty/src/loader.rs b/runtime/rust/prompty/src/loader.rs index 9f730b56a..ba3359deb 100644 --- a/runtime/rust/prompty/src/loader.rs +++ b/runtime/rust/prompty/src/loader.rs @@ -9,6 +9,7 @@ use std::path::{Path, PathBuf}; use crate::model::Agent; use crate::model::context::LoadContext; +pub mod endpoint_trust; mod error; mod frontmatter; mod resolve; @@ -21,6 +22,11 @@ pub struct LoadOptions { /// Additional directories that `${file:...}` references may read from. /// The prompt file's directory is always allowed. pub allowed_file_roots: Vec, + /// Additional origins (e.g. `https://my-proxy.contoso.com`, + /// `https://*.contoso.com`, or `*`) trusted to receive ambient credentials + /// (spec §2.5.1). Built-in provider origins and the comma-separated + /// `PROMPTY_TRUSTED_ENDPOINTS` env var are always included. + pub trusted_endpoints: Vec, } /// Load a `.prompty` file and return a typed `Prompty`. @@ -176,12 +182,24 @@ fn build_agent(raw: &str, file_path: &Path, options: &LoadOptions) -> Result Result Option { + match scheme { + "http" => Some(80), + "https" => Some(443), + _ => None, + } +} + +/// Return `(scheme, host, port)` for an absolute http(s) URL, else `None`. +/// +/// Mirrors Python's `urllib.parse.urlsplit` semantics used by the reference +/// implementation; additionally treats a backslash in the authority as +/// unparseable, since WHATWG-style HTTP clients read it as a path separator. +fn origin(url: &str) -> Option<(String, String, u16)> { + let cleaned: String = url + .trim() + .trim_start_matches(|c: char| c <= ' ') + .chars() + .filter(|c| !matches!(c, '\t' | '\r' | '\n')) + .collect(); + + let colon = cleaned.find(':')?; + let scheme = &cleaned[..colon]; + let mut chars = scheme.chars(); + let first = chars.next()?; + if !first.is_ascii_alphabetic() + || !scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + { + return None; + } + let scheme = scheme.to_ascii_lowercase(); + let default = default_port(&scheme)?; + + let rest = cleaned[colon + 1..].strip_prefix("//")?; + let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let netloc = &rest[..end]; + if netloc.contains('\\') || netloc.contains('[') != netloc.contains(']') { + return None; + } + + let hostinfo = netloc.rsplit_once('@').map_or(netloc, |(_, h)| h); + let (host, port) = match hostinfo.split_once('[') { + Some((_, bracketed)) => { + let (host, after) = bracketed.split_once(']').unwrap_or((bracketed, "")); + if host.parse::().is_err() { + return None; + } + (host, after.split_once(':').map_or("", |(_, p)| p)) + } + None => hostinfo.split_once(':').unwrap_or((hostinfo, "")), + }; + let host = host.to_lowercase(); + if host.is_empty() { + return None; + } + let port = if port.is_empty() { + default + } else { + if !port.chars().all(|c| c.is_ascii_digit()) { + return None; + } + port.parse::().ok()? + }; + Some((scheme, host, port)) +} + +fn trusted_entries(trusted_endpoints: &[String]) -> Vec { + let env = std::env::var(TRUSTED_ENDPOINTS_ENV).unwrap_or_default(); + BUILTIN_TRUSTED_ENDPOINTS + .iter() + .map(|s| s.to_string()) + .chain(trusted_endpoints.iter().cloned()) + .chain(env.split(',').map(str::to_string)) + .map(|e| e.trim().to_string()) + .filter(|e| !e.is_empty()) + .collect() +} + +/// Return `true` if `url`'s origin matches a built-in, host, or env trusted origin. +pub fn is_trusted_endpoint(url: &str, trusted_endpoints: &[String]) -> bool { + let entries = trusted_entries(trusted_endpoints); + if entries.iter().any(|e| e == "*") { + return true; + } + let Some((scheme, host, port)) = origin(url) else { + return false; + }; + entries.iter().any(|entry| { + let Some((a_scheme, pattern, a_port)) = origin(entry) else { + return false; + }; + if a_scheme != scheme || a_port != port { + return false; + } + match pattern.strip_prefix('*') { + Some(suffix) if suffix.starts_with('.') => host.ends_with(suffix), + _ => pattern == host, + } + }) +} + +fn is_reference(value: &str) -> bool { + value.starts_with("${") && value.ends_with('}') +} + +fn file_supplied_credential(connection: &serde_json::Map) -> bool { + // Only the credential field the declared kind actually consumes counts; any other kind is ambient. + let key = match connection.get("kind").and_then(Value::as_str) { + Some("key") => "apiKey", + Some("oauth") => "clientSecret", + _ => return false, + }; + matches!(connection.get(key), Some(Value::String(s)) if !s.is_empty() && !is_reference(s)) +} + +/// Return the effective destination if the file chose it, else `None`. +fn file_controlled_destination(value: Option<&Value>) -> Option { + let value = match value? { + Value::Null => return None, + Value::String(s) => s, + other => return Some(other.to_string()), + }; + if is_reference(value) { + let inner = &value[2..value.len() - 1]; + let (protocol, rest) = inner.split_once(':').unwrap_or((inner, "")); + if protocol.eq_ignore_ascii_case("env") { + let (var_name, default) = rest.split_once(':').unwrap_or((rest, "")); + if std::env::var_os(var_name).is_some() || default.is_empty() { + return None; + } + return Some(default.to_string()); + } + } + Some(value.clone()) +} + +/// Return [`LoadError::UntrustedEndpoint`] if ambient credentials could reach a +/// file-chosen, untrusted destination. +/// +/// `connection` is the RAW (unresolved) `model.connection` value. +pub fn check_connection_trust( + connection: Option<&Value>, + trusted_endpoints: &[String], +) -> Result<(), LoadError> { + let Some(Value::Object(connection)) = connection else { + return Ok(()); + }; + if connection.get("kind").and_then(Value::as_str) == Some("reference") + || file_supplied_credential(connection) + { + return Ok(()); + } + for key in DESTINATION_KEYS { + if let Some(destination) = file_controlled_destination(connection.get(key)) { + if !is_trusted_endpoint(&destination, trusted_endpoints) { + return Err(LoadError::UntrustedEndpoint(format!( + "Connection {key} '{destination}' is not trusted to receive ambient credentials. \ + Set the {key} via ${{env:VAR}}, add its origin to trusted_endpoints or \ + {TRUSTED_ENDPOINTS_ENV}, or use 'kind: reference'." + ))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::loader::{LoadOptions, load_with_options}; + use serial_test::serial; + use std::path::{Path, PathBuf}; + + struct EnvGuard(Vec<(&'static str, Option)>); + + impl EnvGuard { + fn new(vars: &[(&'static str, Option<&str>)]) -> Self { + let mut saved = vec![( + TRUSTED_ENDPOINTS_ENV, + std::env::var_os(TRUSTED_ENDPOINTS_ENV), + )]; + unsafe { std::env::remove_var(TRUSTED_ENDPOINTS_ENV) }; + for (k, v) in vars { + saved.push((k, std::env::var_os(k))); + match v { + Some(v) => unsafe { std::env::set_var(k, v) }, + None => unsafe { std::env::remove_var(k) }, + } + } + EnvGuard(saved) + } + } + + impl Drop for EnvGuard { + fn drop(&mut self) { + for (k, v) in self.0.iter().rev() { + match v { + Some(v) => unsafe { std::env::set_var(k, v) }, + None => unsafe { std::env::remove_var(k) }, + } + } + } + } + + struct TempDir(PathBuf); + + impl TempDir { + fn new(name: &str) -> Self { + let dir = + std::env::temp_dir().join(format!("prompty_trust_{name}_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + TempDir(dir) + } + } + + impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn write(dir: &Path, connection: &str) -> PathBuf { + let p = dir.join("t.prompty"); + std::fs::write( + &p, + format!("---\nname: t\nmodel:\n id: gpt-4\n connection:\n{connection}\n---\nHello\n"), + ) + .unwrap(); + p + } + + fn load_err(p: &Path, trusted: &[&str]) -> LoadError { + let options = LoadOptions { + trusted_endpoints: trusted.iter().map(|s| s.to_string()).collect(), + ..Default::default() + }; + load_with_options(p, options).expect_err("expected load to fail") + } + + fn endpoint(p: &Path, trusted: &[&str]) -> String { + let options = LoadOptions { + trusted_endpoints: trusted.iter().map(|s| s.to_string()).collect(), + ..Default::default() + }; + let agent = load_with_options(p, options).expect("expected load to succeed"); + crate::model_access::model_connection(&agent.model)["endpoint"] + .as_str() + .unwrap_or_default() + .to_string() + } + + fn trusted(url: &str, extra: &[&str]) -> bool { + let extra: Vec = extra.iter().map(|s| s.to_string()).collect(); + is_trusted_endpoint(url, &extra) + } + + #[test] + #[serial(prompty_trusted_env)] + fn builtin_trusted() { + let _env = EnvGuard::new(&[]); + for url in [ + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + ] { + assert!(trusted(url, &[]), "{url} should be trusted"); + } + } + + #[test] + #[serial(prompty_trusted_env)] + fn untrusted() { + let _env = EnvGuard::new(&[]); + for url in [ + "http://api.openai.com", + "https://api.openai.com:8443", + "https://openai.azure.com", + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", + "https://evil.com\\@api.openai.com", + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "https://api.openai.com:99999", + "https://[::1", + "", + ] { + assert!(!trusted(url, &[]), "{url} should be untrusted"); + } + } + + #[test] + #[serial(prompty_trusted_env)] + fn host_option_and_wildcard() { + let _env = EnvGuard::new(&[]); + assert!(trusted( + "http://localhost:11434/v1", + &["http://LOCALHOST:11434"] + )); + assert!(trusted("https://x.contoso.com", &["https://*.contoso.com"])); + assert!(!trusted("https://contoso.com", &["https://*.contoso.com"])); + assert!(trusted("http://[::1]:8080/x", &["http://[::1]:8080"])); + } + + #[test] + #[serial(prompty_trusted_env)] + fn env_list_and_star() { + let _env = EnvGuard::new(&[( + TRUSTED_ENDPOINTS_ENV, + Some(" https://a.example.com , http://localhost:8080"), + )]); + assert!(trusted("http://localhost:8080/x", &[])); + assert!(!trusted("http://localhost:8081", &[])); + unsafe { std::env::set_var(TRUSTED_ENDPOINTS_ENV, "*") }; + assert!(trusted("http://anything.example", &[])); + } + + #[test] + #[serial(prompty_trusted_env)] + fn ambient_literal_endpoint_rejected() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("ambient"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: http://127.0.0.1:9999", + ); + let err = load_err(&p, &[]); + assert!(matches!(err, LoadError::UntrustedEndpoint(_))); + assert!(err.to_string().contains("127.0.0.1")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn env_key_literal_endpoint_rejected() { + let _env = EnvGuard::new(&[("PROMPTY_TRUST_SOME_KEY", Some("secret"))]); + let tmp = TempDir::new("envkey"); + let p = write( + &tmp.0, + " kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:PROMPTY_TRUST_SOME_KEY}", + ); + assert!(load_err(&p, &[]).to_string().contains("not trusted")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn oauth_token_url_checked() { + let _env = EnvGuard::new(&[("PROMPTY_TRUST_SECRET", Some("s"))]); + let tmp = TempDir::new("oauth"); + let p = write( + &tmp.0, + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n \ + clientSecret: ${env:PROMPTY_TRUST_SECRET}\n tokenUrl: https://evil.example.com/token", + ); + assert!(load_err(&p, &[]).to_string().contains("tokenUrl")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn mismatched_credential_field_rejected() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("mismatch"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk", + ); + assert!(load_err(&p, &[]).to_string().contains("not trusted")); + let p = write( + &tmp.0, + " kind: key\n endpoint: https://evil.example.com\n clientSecret: junk", + ); + assert!(load_err(&p, &[]).to_string().contains("not trusted")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn literal_key_allowed() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("literal"); + let p = write( + &tmp.0, + " kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine", + ); + assert_eq!(endpoint(&p, &[]), "http://127.0.0.1:9999"); + } + + #[test] + #[serial(prompty_trusted_env)] + fn env_endpoint_allowed() { + let _env = EnvGuard::new(&[("PROMPTY_TRUST_EP", Some("http://127.0.0.1:9999"))]); + let tmp = TempDir::new("envep"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: ${env:PROMPTY_TRUST_EP}", + ); + assert_eq!(endpoint(&p, &[]), "http://127.0.0.1:9999"); + } + + #[test] + #[serial(prompty_trusted_env)] + fn env_default_used_rejected() { + let _env = EnvGuard::new(&[("PROMPTY_TRUST_UNSET_EP", None)]); + let tmp = TempDir::new("envdefault"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: ${env:PROMPTY_TRUST_UNSET_EP:http://127.0.0.1:9999}", + ); + assert!(load_err(&p, &[]).to_string().contains("not trusted")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn trusted_option_allows() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("option"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: http://127.0.0.1:9999/api", + ); + assert_eq!( + endpoint(&p, &["http://127.0.0.1:9999"]), + "http://127.0.0.1:9999/api" + ); + } + + #[tokio::test] + #[serial(prompty_trusted_env)] + async fn async_honors_option() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("async"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: http://127.0.0.1:9999", + ); + assert!( + crate::loader::load_async(&p).await.is_err(), + "untrusted endpoint must be rejected" + ); + let options = LoadOptions { + trusted_endpoints: vec!["http://127.0.0.1:9999".to_string()], + ..Default::default() + }; + let agent = crate::loader::load_async_with_options(&p, options) + .await + .unwrap(); + assert_eq!( + crate::model_access::model_connection(&agent.model)["endpoint"] + .as_str() + .unwrap_or_default() + .to_string(), + "http://127.0.0.1:9999" + ); + } + + #[test] + #[serial(prompty_trusted_env)] + fn file_model_reference_checked() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("filemodel"); + std::fs::write( + tmp.0.join("model.json"), + r#"{"id": "gpt-4", "connection": {"kind": "foundry", "endpoint": "https://evil.example.com"}}"#, + ) + .unwrap(); + let p = tmp.0.join("t.prompty"); + std::fs::write(&p, "---\nname: t\nmodel: ${file:model.json}\n---\nHi\n").unwrap(); + assert!(load_err(&p, &[]).to_string().contains("not trusted")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn file_connection_traversal_is_file_reference() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("traversal"); + let sub = tmp.0.join("prompts"); + std::fs::create_dir_all(&sub).unwrap(); + std::fs::write( + tmp.0.join("conn.json"), + r#"{"kind": "foundry", "endpoint": "https://evil.example.com"}"#, + ) + .unwrap(); + let p = sub.join("t.prompty"); + std::fs::write( + &p, + "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:../conn.json}\n---\nHi\n", + ) + .unwrap(); + assert!(matches!(load_err(&p, &[]), LoadError::FileReference { .. })); + } + + #[test] + #[serial(prompty_trusted_env)] + fn frontmatter_cannot_self_trust() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("selftrust"); + let p = tmp.0.join("t.prompty"); + std::fs::write( + &p, + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n\ + model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n", + ) + .unwrap(); + assert!(matches!(load_err(&p, &[]), LoadError::UntrustedEndpoint(_))); + } +} diff --git a/runtime/rust/prompty/src/loader/error.rs b/runtime/rust/prompty/src/loader/error.rs index 3e8e68df2..f8f8a2c5a 100644 --- a/runtime/rust/prompty/src/loader/error.rs +++ b/runtime/rust/prompty/src/loader/error.rs @@ -25,6 +25,9 @@ pub enum LoadError { /// Detail message detail: String, }, + /// Ambient credentials would be sent to a file-chosen, untrusted + /// destination (spec §2.5.1). + UntrustedEndpoint(String), /// A catch-all for other load errors (e.g. spawn_blocking panic). Other(String), } @@ -50,6 +53,9 @@ impl std::fmt::Display for LoadError { LoadError::FileReference { path, detail } => { write!(f, "File reference error: {}: {}", path.display(), detail) } + LoadError::UntrustedEndpoint(msg) => { + write!(f, "Untrusted endpoint: {msg}") + } LoadError::Other(msg) => { write!(f, "Load error: {msg}") } diff --git a/runtime/rust/prompty/src/loader/resolve.rs b/runtime/rust/prompty/src/loader/resolve.rs index fdd5139ed..61f91be90 100644 --- a/runtime/rust/prompty/src/loader/resolve.rs +++ b/runtime/rust/prompty/src/loader/resolve.rs @@ -57,6 +57,43 @@ pub fn resolve_single_ref( .flatten() } +/// Follow a whole-value `${file:...}` reference with the same containment +/// rules as full resolution. Non-file values are returned unchanged; a +/// referenced file that does not exist yields `Null` (full resolution reports +/// the error later). +pub fn follow_file_ref( + value: &serde_json::Value, + agent_dir: &Path, + allowed_file_roots: &[PathBuf], +) -> Result { + let Some(s) = value.as_str() else { + return Ok(value.clone()); + }; + if !s.starts_with("${") || !s.ends_with('}') { + return Ok(value.clone()); + } + let inner = &s[2..s.len() - 1]; + let Some((protocol, reference)) = inner.split_once(':') else { + return Ok(value.clone()); + }; + if !protocol.eq_ignore_ascii_case("file") { + return Ok(value.clone()); + } + let requested = Path::new(reference); + let target = if requested.is_absolute() { + requested.to_path_buf() + } else { + agent_dir.join(requested) + }; + if !target.exists() { + return Ok(serde_json::Value::Null); + } + Ok( + resolve_file(reference, agent_dir, allowed_file_roots, "model.connection")? + .unwrap_or(serde_json::Value::Null), + ) +} + /// Try to resolve a single string value. Returns `Some(resolved)` if it was /// a `${protocol:value}` reference, `None` if it should be kept as-is. fn try_resolve_string( diff --git a/runtime/rust/prompty/tests/loader_test.rs b/runtime/rust/prompty/tests/loader_test.rs index d6ee618f8..d07b81510 100644 --- a/runtime/rust/prompty/tests/loader_test.rs +++ b/runtime/rust/prompty/tests/loader_test.rs @@ -181,7 +181,8 @@ fn test_env_default() { "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "${env:MISSING_VAR:fallback_value}" + "endpoint": "${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }); @@ -322,7 +323,17 @@ fn test_connection_types_load() { } } }); - let agent = load_from_frontmatter(&fm, &[]).unwrap(); + let yaml = serde_yaml::to_string(&fm).unwrap(); + let options = prompty::LoadOptions { + trusted_endpoints: vec!["https://localhost:8080".to_string()], + ..Default::default() + }; + let agent = prompty::load_from_string_with_options( + &format!("---\n{yaml}---\n"), + std::env::current_dir().unwrap(), + options, + ) + .unwrap(); let model_conn = prompty::model_access::model_connection(&agent.model); let conn = model_conn.as_object().unwrap(); assert_eq!(conn["kind"], "anonymous"); diff --git a/runtime/rust/prompty/tests/model/vector_adapters.rs b/runtime/rust/prompty/tests/model/vector_adapters.rs index 891d4635c..46887c9a8 100644 --- a/runtime/rust/prompty/tests/model/vector_adapters.rs +++ b/runtime/rust/prompty/tests/model/vector_adapters.rs @@ -750,6 +750,7 @@ fn load_error_payload(err: &prompty::LoadError) -> Value { LE::InvalidTemplate(..) => "invalid_template", LE::EnvVarNotSet { .. } => "env_var_not_set", LE::FileReference { .. } => "file_reference", + LE::UntrustedEndpoint(..) => "untrusted_endpoint", LE::Other(..) => return serde_json::json!({ "message": err.to_string() }), }; serde_json::json!({ "kind": kind }) @@ -843,6 +844,15 @@ fn load_adapter(input: &Value, ctx: &Context) -> Result { } } } + // Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + const TRUST_ENV: &str = "PROMPTY_TRUSTED_ENDPOINTS"; + if !env_vars + .as_ref() + .is_some_and(|vars| vars.contains_key(TRUST_ENV)) + { + restore.push((TRUST_ENV.to_string(), std::env::var(TRUST_ENV).ok())); + unsafe { std::env::remove_var(TRUST_ENV) }; + } let result = load_agent_from_input(&obj, ctx); @@ -859,10 +869,23 @@ fn load_agent_from_input( obj: &serde_json::Map, ctx: &Context, ) -> Result { + let options = prompty::LoadOptions { + trusted_endpoints: obj + .get("trustedEndpoints") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|e| e.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default(), + ..Default::default() + }; + // fixture path if let Some(Value::String(fixture)) = obj.get("fixture") { let path = spec_fixtures(ctx).join(fixture); - return match load(&path) { + return match prompty::load_with_options(&path, options) { Ok(agent) => Ok(agent_to_canonical(&agent)), Err(e) => Err(load_vector_error(e)), }; @@ -895,7 +918,7 @@ fn load_agent_from_input( let prompty_path = agent_dir.join("vector.prompty"); std::fs::write(&prompty_path, text).ok(); - let out = match load(&prompty_path) { + let out = match prompty::load_with_options(&prompty_path, options) { Ok(agent) => Ok(agent_to_canonical(&agent)), Err(e) => Err(load_vector_error(e)), }; From 3acc6b54e76f25a70d1c92db59712f9257dcd6a4 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 12:23:25 -0700 Subject: [PATCH 08/15] feat(swift): enforce endpoint trust at load time Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/swift/README.md | 30 +++ runtime/swift/prompty-model/Info.plist | 16 ++ .../Sources/PromptyModel/EndpointTrust.swift | 192 +++++++++++++++++ .../PromptyModel/PipelineRuntime.swift | 25 ++- .../PromptyModelTests/VectorAdapters.swift | 42 +++- .../prompty-model/model_capabilities.json | 60 ++++++ runtime/swift/prompty/Info.plist | 16 ++ .../Sources/Prompty/EndpointTrust.swift | 29 +++ .../prompty/Sources/Prompty/Errors.swift | 5 + .../prompty/Sources/Prompty/Loader.swift | 21 +- .../PromptyTests/EndpointTrustTests.swift | 202 ++++++++++++++++++ .../Tests/PromptyTests/LoadVectorTests.swift | 25 ++- .../PromptyTests/ReadmeSnippetTests.swift | 8 + runtime/swift/prompty/model_capabilities.json | 60 ++++++ 14 files changed, 705 insertions(+), 26 deletions(-) create mode 100644 runtime/swift/prompty-model/Info.plist create mode 100644 runtime/swift/prompty-model/Sources/PromptyModel/EndpointTrust.swift create mode 100644 runtime/swift/prompty-model/model_capabilities.json create mode 100644 runtime/swift/prompty/Info.plist create mode 100644 runtime/swift/prompty/Sources/Prompty/EndpointTrust.swift create mode 100644 runtime/swift/prompty/Tests/PromptyTests/EndpointTrustTests.swift create mode 100644 runtime/swift/prompty/model_capabilities.json diff --git a/runtime/swift/README.md b/runtime/swift/README.md index 3a0eb49df..ca27dba24 100644 --- a/runtime/swift/README.md +++ b/runtime/swift/README.md @@ -55,6 +55,36 @@ let messages = try await Pipeline.prepare(agent, inputs: inputs) // render + pa let raw = try await Pipeline.run(agent, messages: messages) // execute + process ``` +### Endpoint trust + +A `.prompty` file is untrusted input. If a connection uses ambient credentials +(an `${env:}` API key, a bearer token, a managed identity), the loader refuses a +literal `endpoint` or `tokenUrl` that the host hasn't approved. It throws +`LoadError.untrustedEndpoint` (spec §2.5.1). By default the loader trusts these +origins: + +- `https://api.openai.com` +- `https://api.anthropic.com` +- `https://*.openai.azure.com` +- `https://*.services.ai.azure.com` +- `https://*.cognitiveservices.azure.com` +- `https://login.microsoftonline.com` + +To trust other origins, pass them as a load option or set them in the +environment: + +```swift +let agent = try Loader.load( + path: "local.prompty", + options: LoadOptions(trustedEndpoints: ["http://localhost:11434", "https://*.contoso.com"]) +) +``` + +`PROMPTY_TRUSTED_ENDPOINTS` is a comma-separated list of origins and is merged +with the option. `*` trusts everything. The check passes without any trust entry +when the endpoint comes from `${env:VAR}` (host-controlled) or when the file +carries its own literal key. + ### Tool calls When a prompt declares tools, the host drives the loop. Read the calls, then ask diff --git a/runtime/swift/prompty-model/Info.plist b/runtime/swift/prompty-model/Info.plist new file mode 100644 index 000000000..c33b7789a --- /dev/null +++ b/runtime/swift/prompty-model/Info.plist @@ -0,0 +1,16 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleIdentifier + prompty-model.PromptyModel.resources + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + PromptyModel_PromptyModel + CFBundlePackageType + BNDL + + diff --git a/runtime/swift/prompty-model/Sources/PromptyModel/EndpointTrust.swift b/runtime/swift/prompty-model/Sources/PromptyModel/EndpointTrust.swift new file mode 100644 index 000000000..244c7a2ca --- /dev/null +++ b/runtime/swift/prompty-model/Sources/PromptyModel/EndpointTrust.swift @@ -0,0 +1,192 @@ +import Foundation + +/// Endpoint trust — keep ambient credentials on host-approved destinations (spec §2.5.1). +/// +/// A `.prompty` file is untrusted input. When its connection carries no +/// file-supplied credential, executors fall back to ambient credentials +/// (Azure identity tokens, `*_API_KEY` env vars). This decides whether the +/// file-chosen destination may receive them. +public enum EndpointTrust { + /// Comma-separated list of extra trusted origins supplied by the host environment. + public static let trustedEndpointsEnv = "PROMPTY_TRUSTED_ENDPOINTS" + + /// First-party provider origins trusted without host configuration. + public static let builtinTrustedEndpoints: [String] = [ + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com", + ] + + private static let defaultPorts: [String: Int] = ["http": 80, "https": 443] + private static let credentialKeys: [String: String] = ["key": "apiKey", "oauth": "clientSecret"] + private static let destinationKeys = ["endpoint", "tokenUrl"] + + /// Return `true` if `url`'s origin matches a built-in, host, or env trusted origin. + public static func isTrusted(_ url: String, trustedEndpoints: [String] = []) -> Bool { + let entries = trustedEntries(trustedEndpoints) + if entries.contains("*") { return true } + guard let target = origin(url) else { return false } + for entry in entries { + guard let allowed = origin(entry), allowed.scheme == target.scheme, allowed.port == target.port else { + continue + } + if allowed.host.hasPrefix("*.") { + if target.host.hasSuffix(String(allowed.host.dropFirst())) { return true } + } else if allowed.host == target.host { + return true + } + } + return false + } + + /// Throw `untrusted_endpoint` if ambient credentials could reach a file-chosen, untrusted destination. + /// + /// `connection` is the RAW (unresolved) `model.connection` mapping. + public static func checkConnection(_ connection: Any?, trustedEndpoints: [String] = []) throws { + guard let connection = connection as? [String: Any] else { return } + if connection["kind"] as? String == "reference" { return } + if fileSuppliedCredential(connection) { return } + for key in destinationKeys { + guard let destination = fileControlledDestination(connection[key]), + !isTrusted(destination, trustedEndpoints: trustedEndpoints) + else { continue } + throw PromptyLoadError( + kind: "untrusted_endpoint", + message: "Connection \(key) '\(destination)' is not trusted to receive ambient credentials. " + + "Set the \(key) via ${env:VAR}, add its origin to trustedEndpoints or " + + "\(trustedEndpointsEnv), or use 'kind: reference'.") + } + } + + /// Return the unresolved `model.connection`, following whole-value `${file:}` references. + /// + /// `resolveFile` receives the reference path and returns the loaded content, or + /// `nil` when the file does not exist (resolution reports that later). + public static func rawConnection( + _ data: [String: Any], resolveFile: (String) throws -> Any? + ) rethrows -> Any? { + func follow(_ value: Any?) throws -> Any? { + if let string = value as? String, isReference(string) { + let inner = String(string.dropFirst(2).dropLast()) + if let colon = inner.firstIndex(of: ":"), inner[.. String? { + if promptyEnvironmentOverrides.keys.contains(name) { + return promptyEnvironmentOverrides[name] ?? nil + } + return ProcessInfo.processInfo.environment[name] + } + + private static func trustedEntries(_ trustedEndpoints: [String]) -> [String] { + var entries = builtinTrustedEndpoints + trustedEndpoints + entries.append( + contentsOf: (environmentValue(trustedEndpointsEnv) ?? "").split(separator: ",", omittingEmptySubsequences: false) + .map(String.init)) + return entries.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty } + } + + private static func isReference(_ value: String) -> Bool { + value.hasPrefix("${") && value.hasSuffix("}") + } + + private static func fileSuppliedCredential(_ connection: [String: Any]) -> Bool { + // Only the credential field the declared kind actually consumes counts; any other kind is ambient. + guard let kind = connection["kind"] as? String, let key = credentialKeys[kind], + let value = connection[key] as? String + else { return false } + return !value.isEmpty && !isReference(value) + } + + /// Return the effective destination if the file chose it, else `nil`. + private static func fileControlledDestination(_ value: Any?) -> String? { + guard let value, !(value is NSNull) else { return nil } + guard let string = value as? String else { return "\(value)" } + if isReference(string) { + let inner = String(string.dropFirst(2).dropLast()) + let parts = inner.split(separator: ":", maxSplits: 1, omittingEmptySubsequences: false) + if parts[0].lowercased() == "env" { + let rest = parts.count > 1 ? String(parts[1]) : "" + let pieces = rest.split(separator: ":", maxSplits: 1, omittingEmptySubsequences: false) + let name = String(pieces[0]) + if environmentValue(name) != nil { return nil } + let fallback = pieces.count > 1 ? String(pieces[1]) : "" + return fallback.isEmpty ? nil : fallback + } + } + return string + } + + struct Origin: Equatable { + let scheme: String + let host: String + let port: Int + } + + /// Return `(scheme, host, port)` for an absolute http(s) URL, else `nil`. + /// + /// Mirrors Python's `urllib.parse.urlsplit` so every runtime classifies the same strings alike. + static func origin(_ url: String) -> Origin? { + let trimmed = url.trimmingCharacters(in: .whitespacesAndNewlines) + let text = String(String.UnicodeScalarView(trimmed.unicodeScalars.filter { !"\t\r\n".unicodeScalars.contains($0) })) + guard let colon = text.firstIndex(of: ":") else { return nil } + let rawScheme = text[.. Agent { return try Agent.load(data) } -public func loadPromptyContent(_ contents: String, parentDir: URL, allowedRoots: [URL]) throws -> Agent { +/// Enforce endpoint trust (spec §2.5.1) on the RAW frontmatter, before references resolve. +/// +/// Whole-value `${file:}` references for `model` and `model.connection` are followed +/// with the same containment as reference resolution. +public func checkEndpointTrust( + _ data: [String: Any], parentDir: URL, allowedRoots: [URL], trustedEndpoints: [String] = [] +) throws { + let connection = try EndpointTrust.rawConnection(data) { reference in + guard FileManager.default.fileExists(atPath: relativeFileURL(reference, base: parentDir).path) else { + return nil + } + return try resolveFile(reference, parentDir: parentDir, allowedRoots: allowedRoots) + } + try EndpointTrust.checkConnection(connection, trustedEndpoints: trustedEndpoints) +} + +public func loadPromptyContent( + _ contents: String, parentDir: URL, allowedRoots: [URL], trustedEndpoints: [String] = [] +) throws -> Agent { var data = try parseFrontmatter(contents) + try checkEndpointTrust(data, parentDir: parentDir, allowedRoots: allowedRoots, trustedEndpoints: trustedEndpoints) try resolveReferences(&data, parentDir: parentDir, allowedRoots: allowedRoots) return try buildAgentFromData(data) } -public func loadPromptyFile(_ path: URL) throws -> Agent { +public func loadPromptyFile(_ path: URL, trustedEndpoints: [String] = []) throws -> Agent { guard FileManager.default.fileExists(atPath: path.path) else { throw PromptyLoadError(kind: "not_found", message: "FileNotFoundError: prompty file not found: '\(path.path)'") } let text = (try String(contentsOf: path, encoding: .utf8)).replacingOccurrences(of: "\r\n", with: "\n") let dir = canonicalURL(path.deletingLastPathComponent()) - return try loadPromptyContent(text, parentDir: dir, allowedRoots: [dir]) + return try loadPromptyContent(text, parentDir: dir, allowedRoots: [dir], trustedEndpoints: trustedEndpoints) } public func validateInputs(agent: Agent, provided: [String: Any]) throws -> [String: Any] { diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift index 386230829..263b3a64f 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift @@ -313,8 +313,10 @@ enum VectorAdapters { static func loadInvoke(_ input: Any?, _ context: VectorContext) throws -> Any? { let object = input as? [String: Any] ?? [:] - let restore = applyEnv(object["env"]) + let env = object["env"] as? [String: Any] ?? [:] + let restore = applyEnv(env, unset: env[EndpointTrust.trustedEndpointsEnv] == nil ? [EndpointTrust.trustedEndpointsEnv] : []) defer { restore() } + let trusted = (object["trustedEndpoints"] as? [Any] ?? []).compactMap { $0 as? String } // Input-validation vectors carry inputs alongside frontmatter at the top level; // full-load vectors nest inputs inside the frontmatter. @@ -330,11 +332,11 @@ enum VectorAdapters { do { let agent: Agent if let fixture = object["fixture"] as? String, !fixture.isEmpty { - agent = try loadPromptyFile(findSpecFixtures(context.baseDir).appendingPathComponent(fixture)) + agent = try loadPromptyFile(findSpecFixtures(context.baseDir).appendingPathComponent(fixture), trustedEndpoints: trusted) } else if let raw = object["frontmatter_raw"] as? String, !raw.isEmpty { - agent = try loadFromRaw(raw) + agent = try loadFromRaw(raw, trustedEndpoints: trusted) } else if object["frontmatter"] is [String: Any] { - agent = try materializeAndLoad(object) + agent = try materializeAndLoad(object, trustedEndpoints: trusted) } else { return ["error": ""] } @@ -361,13 +363,15 @@ enum VectorAdapters { return root } - static func loadFromRaw(_ raw: String) throws -> Agent { + static func loadFromRaw(_ raw: String, trustedEndpoints: [String] = []) throws -> Agent { let dir = try scratchDir("prompty-load-") defer { try? FileManager.default.removeItem(at: dir) } - return try loadPromptyContent(raw.replacingOccurrences(of: "\r\n", with: "\n"), parentDir: dir, allowedRoots: [dir]) + return try loadPromptyContent( + raw.replacingOccurrences(of: "\r\n", with: "\n"), parentDir: dir, allowedRoots: [dir], + trustedEndpoints: trustedEndpoints) } - static func materializeAndLoad(_ input: [String: Any]) throws -> Agent { + static func materializeAndLoad(_ input: [String: Any], trustedEndpoints: [String] = []) throws -> Agent { let tempBase = try scratchDir("prompty-load-") defer { try? FileManager.default.removeItem(at: tempBase) } var agentDir = tempBase @@ -382,6 +386,7 @@ enum VectorAdapters { } var frontmatter = input["frontmatter"] as? [String: Any] ?? [:] let root = agentDir.standardizedFileURL + try checkEndpointTrust(frontmatter, parentDir: agentDir, allowedRoots: [root], trustedEndpoints: trustedEndpoints) try resolveReferences(&frontmatter, parentDir: agentDir, allowedRoots: [root]) return try buildAgentFromData(frontmatter) } @@ -455,16 +460,31 @@ enum VectorAdapters { return payload } - static func applyEnv(_ value: Any?) -> () -> Void { + /// Apply a vector's `env` (and force-unset `unset`) through `promptyEnvironmentOverrides`, + /// returning a closure that restores the prior override state exactly. + static func applyEnv(_ value: Any?, unset: [String] = []) -> () -> Void { let env = value as? [String: Any] ?? [:] - var saved: [String: String?] = [:] + var saved: [String: String??] = [:] + func save(_ key: String) { + if !saved.keys.contains(key) { + saved[key] = promptyEnvironmentOverrides.keys.contains(key) ? .some(promptyEnvironmentOverrides[key] ?? nil) : .none + } + } for (key, raw) in env { - saved[key] = promptyEnvironmentOverrides[key] ?? nil + save(key) promptyEnvironmentOverrides[key] = "\(raw)" } + for key in unset { + save(key) + promptyEnvironmentOverrides[key] = .some(nil) + } return { for (key, previous) in saved { - promptyEnvironmentOverrides[key] = previous + if let previous { + promptyEnvironmentOverrides[key] = previous + } else { + promptyEnvironmentOverrides.removeValue(forKey: key) + } } } } diff --git a/runtime/swift/prompty-model/model_capabilities.json b/runtime/swift/prompty-model/model_capabilities.json new file mode 100644 index 000000000..6951f344f --- /dev/null +++ b/runtime/swift/prompty-model/model_capabilities.json @@ -0,0 +1,60 @@ +{ + "description": "Cross-runtime fallback capability data for provider model discovery. Some provider /models endpoints (Anthropic, Foundry) return capability fields directly; others (OpenAI) return only ids. To keep discovery results consistent across providers AND across runtimes, every Prompty runtime embeds THIS file and applies one shared rule: provider-supplied fields always win; entries here only fill fields the provider left empty (fill-only-missing). Model ids are matched by longest prefix within a provider's list. This dataset is deliberately NOT emitted from TypeSpec: it is volatile provider data (context windows, modalities, new model families) refreshed as a snapshot, whereas TypeSpec owns the structural ModelInfo contract. Fields use the canonical camelCase ModelInfo names. A missing 'contextWindow' means unknown; a present empty modality list (e.g. []) is intentional (e.g. embeddings produce no textual/image output modality).", + "match": "longest_prefix", + "providers": { + "openai": [ + { + "prefix": "gpt-4o-mini", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4o", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4-turbo", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4", + "contextWindow": 8192, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-3.5-turbo", + "contextWindow": 16385, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "text-embedding-3-small", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "text-embedding-3-large", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "gpt-image-1", + "inputModalities": ["text", "image"], + "outputModalities": ["image"] + }, + { + "prefix": "dall-e-3", + "inputModalities": ["text"], + "outputModalities": ["image"] + } + ] + } +} diff --git a/runtime/swift/prompty/Info.plist b/runtime/swift/prompty/Info.plist new file mode 100644 index 000000000..3eaf43945 --- /dev/null +++ b/runtime/swift/prompty/Info.plist @@ -0,0 +1,16 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleIdentifier + prompty.Prompty.resources + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Prompty_Prompty + CFBundlePackageType + BNDL + + diff --git a/runtime/swift/prompty/Sources/Prompty/EndpointTrust.swift b/runtime/swift/prompty/Sources/Prompty/EndpointTrust.swift new file mode 100644 index 000000000..5daceaa9a --- /dev/null +++ b/runtime/swift/prompty/Sources/Prompty/EndpointTrust.swift @@ -0,0 +1,29 @@ +import Foundation +import PromptyModel + +/// Loader-side wiring for endpoint trust (spec §2.5.1). +/// +/// The classification and origin matching live in `PromptyModel.EndpointTrust`; +/// this extracts the RAW `model.connection` (following whole-value `${file:}` +/// references for `model` and `model.connection` with the same containment as +/// reference resolution) and maps a rejection onto `LoadError.untrustedEndpoint`. +enum EndpointTrustCheck { + + static func check( + _ data: [String: Any], agentDirectory: URL, allowedRoots: [URL], trustedEndpoints: [String] + ) throws { + let connection = try EndpointTrust.rawConnection(data) { reference -> Any? in + let target = URL(fileURLWithPath: reference, relativeTo: agentDirectory) + .standardizedFileURL.resolvingSymlinksInPath() + // A missing file is reported by reference resolution, after this check. + guard FileManager.default.fileExists(atPath: target.path) else { return nil } + return try References.resolveString( + "${file:\(reference)}", key: "model", agentDirectory: agentDirectory, allowedRoots: allowedRoots) + } + do { + try EndpointTrust.checkConnection(connection, trustedEndpoints: trustedEndpoints) + } catch let error as PromptyLoadError { + throw LoadError.untrustedEndpoint(error.message) + } + } +} diff --git a/runtime/swift/prompty/Sources/Prompty/Errors.swift b/runtime/swift/prompty/Sources/Prompty/Errors.swift index dba5c6fef..114b19e51 100644 --- a/runtime/swift/prompty/Sources/Prompty/Errors.swift +++ b/runtime/swift/prompty/Sources/Prompty/Errors.swift @@ -19,6 +19,9 @@ public enum LoadError: Error, CustomStringConvertible, Equatable { /// value that can never be a property. `path` is the full dotted path to the /// offending entry and `valueCategory` names the rejected shape. case invalidNamedCollectionEntry(path: String, valueCategory: String) + /// The connection would send ambient credentials to a destination the file + /// chose that the host has not trusted (spec §2.5.1). + case untrustedEndpoint(String) public var description: String { switch self { @@ -37,6 +40,8 @@ public enum LoadError: Error, CustomStringConvertible, Equatable { "invalid-named-collection-entry: \(path) holds a \(valueCategory), " + "which is not a valid named collection entry. Declare the array inside " + "a property instead, for example `\(path): { kind: array, default: [...] }`" + case .untrustedEndpoint(let message): + return message } } } diff --git a/runtime/swift/prompty/Sources/Prompty/Loader.swift b/runtime/swift/prompty/Sources/Prompty/Loader.swift index 36d801746..a72467734 100644 --- a/runtime/swift/prompty/Sources/Prompty/Loader.swift +++ b/runtime/swift/prompty/Sources/Prompty/Loader.swift @@ -12,8 +12,14 @@ public struct LoadOptions { /// file's own directory is always permitted. public var allowedFileRoots: [URL] - public init(allowedFileRoots: [URL] = []) { + /// Extra origins (e.g. `https://my-proxy.contoso.com`, `https://*.contoso.com`, + /// or `*`) trusted to receive ambient credentials, in addition to the built-in + /// provider origins and `PROMPTY_TRUSTED_ENDPOINTS` (spec §2.5.1). + public var trustedEndpoints: [String] + + public init(allowedFileRoots: [URL] = [], trustedEndpoints: [String] = []) { self.allowedFileRoots = allowedFileRoots + self.trustedEndpoints = trustedEndpoints } public static let `default` = LoadOptions() @@ -84,8 +90,13 @@ public enum Loader { // 3. A `.prompty` file always describes a prompt. data["kind"] = Defaults.kind - // 4. Resolve ${env:} / ${file:} before handing the tree to the model. + // 4. Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. let agentDirectory = filePath.deletingLastPathComponent() + try EndpointTrustCheck.check( + data, agentDirectory: agentDirectory, allowedRoots: options.allowedFileRoots, + trustedEndpoints: options.trustedEndpoints) + + // 5. Resolve ${env:} / ${file:} before handing the tree to the model. var tree: Any = data try References.resolve( &tree, agentDirectory: agentDirectory, allowedRoots: options.allowedFileRoots) @@ -94,12 +105,12 @@ public enum Loader { throw LoadError.invalidFrontmatter("Frontmatter must be a YAML mapping") } - // 5. Normalize the shapes the schema accepts as shorthand. + // 6. Normalize the shapes the schema accepts as shorthand. normalizeModel(&resolved) try normalizeProperties(&resolved, key: "inputs", path: "inputs") try normalizeProperties(&resolved, key: "outputs", path: "outputs") - // 6. Hand off to the generated model — the canonical type layer. + // 7. Hand off to the generated model — the canonical type layer. var agent: Agent do { agent = try Agent.load(resolved) @@ -107,7 +118,7 @@ public enum Loader { throw LoadError.invalidModel(String(describing: error)) } - // 7. Record the source path so relative tool references can resolve later. + // 8. Record the source path so relative tool references can resolve later. var metadata = agent.metadata ?? [:] metadata[Defaults.sourcePathKey] = filePath.path agent.metadata = metadata diff --git a/runtime/swift/prompty/Tests/PromptyTests/EndpointTrustTests.swift b/runtime/swift/prompty/Tests/PromptyTests/EndpointTrustTests.swift new file mode 100644 index 000000000..1c68da592 --- /dev/null +++ b/runtime/swift/prompty/Tests/PromptyTests/EndpointTrustTests.swift @@ -0,0 +1,202 @@ +import Foundation +import PromptyModel +import XCTest + +@testable import Prompty + +/// Endpoint trust (spec §2.5.1): ambient credentials only go to host-approved destinations. +final class EndpointTrustTests: XCTestCase { + + private var directory: URL! + private var savedEnv: [String: String?] = [:] + + override func setUpWithError() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("prompty_trust_\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + setEnv(EndpointTrust.trustedEndpointsEnv, nil) + } + + override func tearDownWithError() throws { + for (key, value) in savedEnv { Env.set(key, value) } + savedEnv = [:] + try? FileManager.default.removeItem(at: directory) + } + + private func setEnv(_ key: String, _ value: String?) { + if !savedEnv.keys.contains(key) { + savedEnv.updateValue(ProcessInfo.processInfo.environment[key], forKey: key) + } + Env.set(key, value) + } + + @discardableResult + private func write(_ connection: String, name: String = "t.prompty") throws -> String { + let path = directory.appendingPathComponent(name) + try "---\nname: t\nmodel:\n id: gpt-4\n connection:\n\(connection)\n---\nHello\n" + .write(to: path, atomically: true, encoding: .utf8) + return path.path + } + + private func assertUntrusted( + _ body: @autoclosure () throws -> Agent, contains fragment: String = "not trusted", + file: StaticString = #filePath, line: UInt = #line + ) { + XCTAssertThrowsError(try body(), file: file, line: line) { error in + guard case .untrustedEndpoint(let message) = error as? LoadError else { + return XCTFail("expected LoadError.untrustedEndpoint, got \(error)", file: file, line: line) + } + XCTAssertTrue(message.contains(fragment), "'\(message)' lacks '\(fragment)'", file: file, line: line) + } + } + + private func endpoint(_ agent: Agent) -> String? { + switch agent.model?.connection { + case .apiKeyConnection(let c): return c.endpoint + case .foundryConnection(let c): return c.endpoint + default: return nil + } + } + + // MARK: - isTrusted + + func testBuiltinTrusted() { + for url in [ + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + ] { + XCTAssertTrue(EndpointTrust.isTrusted(url), url) + } + } + + func testUntrusted() { + for url in [ + "http://api.openai.com", + "https://api.openai.com:8443", + "https://openai.azure.com", + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "", + ] { + XCTAssertFalse(EndpointTrust.isTrusted(url), url) + } + } + + func testHostOptionAndWildcard() { + XCTAssertTrue(EndpointTrust.isTrusted("http://localhost:11434/v1", trustedEndpoints: ["http://LOCALHOST:11434"])) + XCTAssertTrue(EndpointTrust.isTrusted("https://x.contoso.com", trustedEndpoints: ["https://*.contoso.com"])) + XCTAssertFalse(EndpointTrust.isTrusted("https://contoso.com", trustedEndpoints: ["https://*.contoso.com"])) + } + + func testEnvListAndStar() { + setEnv(EndpointTrust.trustedEndpointsEnv, " https://a.example.com , http://localhost:8080") + XCTAssertTrue(EndpointTrust.isTrusted("http://localhost:8080/x")) + XCTAssertFalse(EndpointTrust.isTrusted("http://localhost:8081")) + setEnv(EndpointTrust.trustedEndpointsEnv, "*") + XCTAssertTrue(EndpointTrust.isTrusted("http://anything.example")) + } + + // MARK: - Loader enforcement + + func testAmbientLiteralEndpointRejected() throws { + let path = try write(" kind: foundry\n endpoint: http://127.0.0.1:9999") + assertUntrusted(try Loader.load(path: path), contains: "127.0.0.1") + } + + func testEnvKeyLiteralEndpointRejected() throws { + setEnv("SOME_KEY", "secret") + let path = try write(" kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}") + assertUntrusted(try Loader.load(path: path)) + } + + func testOAuthTokenUrlChecked() throws { + setEnv("SECRET", "s") + let path = try write( + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token") + assertUntrusted(try Loader.load(path: path), contains: "tokenUrl") + } + + func testMismatchedCredentialFieldRejected() throws { + let foundry = try write(" kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk") + assertUntrusted(try Loader.load(path: foundry)) + let key = try write(" kind: key\n endpoint: https://evil.example.com\n clientSecret: junk") + assertUntrusted(try Loader.load(path: key)) + } + + func testLiteralKeyAllowed() throws { + let path = try write(" kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine") + XCTAssertEqual(endpoint(try Loader.load(path: path)), "http://127.0.0.1:9999") + } + + func testEnvEndpointAllowed() throws { + setEnv("EP", "http://127.0.0.1:9999") + let path = try write(" kind: foundry\n endpoint: ${env:EP}") + XCTAssertEqual(endpoint(try Loader.load(path: path)), "http://127.0.0.1:9999") + } + + func testEnvDefaultUsedRejected() throws { + setEnv("UNSET_EP", nil) + let path = try write(" kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}") + assertUntrusted(try Loader.load(path: path)) + } + + func testTrustedOptionAllows() throws { + let path = try write(" kind: foundry\n endpoint: http://127.0.0.1:9999/api") + let agent = try Loader.load(path: path, options: LoadOptions(trustedEndpoints: ["http://127.0.0.1:9999"])) + XCTAssertEqual(endpoint(agent), "http://127.0.0.1:9999/api") + } + + func testAsyncHonorsOption() async throws { + let path = try write(" kind: foundry\n endpoint: http://127.0.0.1:9999") + do { + _ = try await Loader.loadAsync(path: path) + XCTFail("expected untrusted_endpoint") + } catch LoadError.untrustedEndpoint { + } + let agent = try await Loader.loadAsync( + path: path, options: LoadOptions(trustedEndpoints: ["http://127.0.0.1:9999"])) + XCTAssertEqual(endpoint(agent), "http://127.0.0.1:9999") + } + + func testFileModelReferenceChecked() throws { + let model: [String: Any] = [ + "id": "gpt-4", "connection": ["kind": "foundry", "endpoint": "https://evil.example.com"], + ] + try JSONSerialization.data(withJSONObject: model).write(to: directory.appendingPathComponent("model.json")) + let path = directory.appendingPathComponent("t.prompty") + try "---\nname: t\nmodel: ${file:model.json}\n---\nHi\n".write(to: path, atomically: true, encoding: .utf8) + assertUntrusted(try Loader.load(path: path.path)) + } + + func testFileTraversalStillReportsFileReference() throws { + let app = directory.appendingPathComponent("app") + try FileManager.default.createDirectory(at: app, withIntermediateDirectories: true) + let secret: [String: Any] = ["kind": "foundry", "endpoint": "https://evil.example.com"] + try JSONSerialization.data(withJSONObject: secret).write(to: directory.appendingPathComponent("secret.json")) + let path = app.appendingPathComponent("t.prompty") + try "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:../secret.json}\n---\nHi\n" + .write(to: path, atomically: true, encoding: .utf8) + XCTAssertThrowsError(try Loader.load(path: path.path)) { error in + guard case .fileReference = error as? LoadError else { + return XCTFail("expected LoadError.fileReference, got \(error)") + } + } + } + + func testFrontmatterCannotSelfTrust() throws { + let path = directory.appendingPathComponent("t.prompty") + try ( + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n" + ).write(to: path, atomically: true, encoding: .utf8) + assertUntrusted(try Loader.load(path: path.path)) + } +} diff --git a/runtime/swift/prompty/Tests/PromptyTests/LoadVectorTests.swift b/runtime/swift/prompty/Tests/PromptyTests/LoadVectorTests.swift index 413c5a570..a4e10dfb3 100644 --- a/runtime/swift/prompty/Tests/PromptyTests/LoadVectorTests.swift +++ b/runtime/swift/prompty/Tests/PromptyTests/LoadVectorTests.swift @@ -142,7 +142,11 @@ final class LoadVectorTests: XCTestCase { private static func runVector( name: String, input: [String: Any], expected: [String: Any], expectedError: [String: Any]? ) throws { - let env = input["env"] as? [String: Any] ?? [:] + var env = input["env"] as? [String: Any] ?? [:] + // Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + if env[EndpointTrust.trustedEndpointsEnv] == nil { + env[EndpointTrust.trustedEndpointsEnv] = NSNull() + } return try withEnvironment(env) { // Vectors that declare a canonical `expectedError` drive the loader (or @@ -167,8 +171,11 @@ final class LoadVectorTests: XCTestCase { /// Build a `Agent` from whichever input form the vector uses. static func loadAgent(_ input: [String: Any]) throws -> Agent { + let options = LoadOptions( + trustedEndpoints: (input["trustedEndpoints"] as? [Any] ?? []).compactMap { $0 as? String }) + if let fixture = input["fixture"] as? String { - return try Loader.load(path: Spec.fixtures.appendingPathComponent(fixture).path) + return try Loader.load(path: Spec.fixtures.appendingPathComponent(fixture).path, options: options) } if let files = input["files"] as? [String: Any] { @@ -203,18 +210,19 @@ final class LoadVectorTests: XCTestCase { let raw = try promptyDocument(frontmatter: input["frontmatter"]) return try Loader.load( - contents: raw, basePath: agentDir.appendingPathComponent("virtual.prompty").path) + contents: raw, basePath: agentDir.appendingPathComponent("virtual.prompty").path, + options: options) } if let raw = input["frontmatter_raw"] as? String { - return try Loader.load(contents: raw, basePath: workingFile) + return try Loader.load(contents: raw, basePath: workingFile, options: options) } guard input["frontmatter"] != nil else { throw VectorFailure("vector has no fixture, frontmatter, or frontmatter_raw") } let raw = try promptyDocument(frontmatter: input["frontmatter"]) - return try Loader.load(contents: raw, basePath: workingFile) + return try Loader.load(contents: raw, basePath: workingFile, options: options) } /// Serialize a vector's `frontmatter` object into a `.prompty` document. @@ -324,6 +332,7 @@ final class LoadVectorTests: XCTestCase { case .invalidFrontmatter: return ["kind": "invalid_frontmatter"] case .invalidModel: return ["kind": "invalid_model"] case .invalidNamedCollectionEntry: return ["kind": "invalid_named_collection_entry"] + case .untrustedEndpoint: return ["kind": "untrusted_endpoint"] } } @@ -642,8 +651,10 @@ final class LoadVectorTests: XCTestCase { func withEnvironment(_ values: [String: Any], _ body: () throws -> T) rethrows -> T { var restore: [String: String?] = [:] for (key, value) in values { - guard let string = value as? String else { continue } - restore[key] = ProcessInfo.processInfo.environment[key] + // `NSNull` unsets the variable for the duration of `body`. + let string = value as? String + guard string != nil || value is NSNull else { continue } + restore.updateValue(ProcessInfo.processInfo.environment[key], forKey: key) Env.set(key, string) } defer { diff --git a/runtime/swift/prompty/Tests/PromptyTests/ReadmeSnippetTests.swift b/runtime/swift/prompty/Tests/PromptyTests/ReadmeSnippetTests.swift index 0dd50bbde..8f2f61a72 100644 --- a/runtime/swift/prompty/Tests/PromptyTests/ReadmeSnippetTests.swift +++ b/runtime/swift/prompty/Tests/PromptyTests/ReadmeSnippetTests.swift @@ -33,6 +33,13 @@ final class ReadmeSnippetTests: XCTestCase { return raw } + func trusted() throws -> Agent { + try Loader.load( + path: "local.prompty", + options: LoadOptions(trustedEndpoints: ["http://localhost:11434", "https://*.contoso.com"]) + ) + } + func toolLoop( agent: Agent, raw: Any?, inputs: [String: Any], myTools: [String: ([String: Any]) throws -> String] @@ -48,6 +55,7 @@ final class ReadmeSnippetTests: XCTestCase { XCTAssertNotNil(quickStart) XCTAssertNotNil(stages) + XCTAssertNotNil(trusted) XCTAssertNotNil(toolLoop) } } diff --git a/runtime/swift/prompty/model_capabilities.json b/runtime/swift/prompty/model_capabilities.json new file mode 100644 index 000000000..6951f344f --- /dev/null +++ b/runtime/swift/prompty/model_capabilities.json @@ -0,0 +1,60 @@ +{ + "description": "Cross-runtime fallback capability data for provider model discovery. Some provider /models endpoints (Anthropic, Foundry) return capability fields directly; others (OpenAI) return only ids. To keep discovery results consistent across providers AND across runtimes, every Prompty runtime embeds THIS file and applies one shared rule: provider-supplied fields always win; entries here only fill fields the provider left empty (fill-only-missing). Model ids are matched by longest prefix within a provider's list. This dataset is deliberately NOT emitted from TypeSpec: it is volatile provider data (context windows, modalities, new model families) refreshed as a snapshot, whereas TypeSpec owns the structural ModelInfo contract. Fields use the canonical camelCase ModelInfo names. A missing 'contextWindow' means unknown; a present empty modality list (e.g. []) is intentional (e.g. embeddings produce no textual/image output modality).", + "match": "longest_prefix", + "providers": { + "openai": [ + { + "prefix": "gpt-4o-mini", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4o", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4-turbo", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4", + "contextWindow": 8192, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-3.5-turbo", + "contextWindow": 16385, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "text-embedding-3-small", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "text-embedding-3-large", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "gpt-image-1", + "inputModalities": ["text", "image"], + "outputModalities": ["image"] + }, + { + "prefix": "dall-e-3", + "inputModalities": ["text"], + "outputModalities": ["image"] + } + ] + } +} From 019785db9a8899bd9f0710ec438440462f727742 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 12:30:17 -0700 Subject: [PATCH 09/15] feat(csharp): enforce endpoint trust at load time Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/csharp/CHANGELOG.md | 4 + .../Prompty.Core.Tests/EndpointTrustTests.cs | 202 ++++++++++++++ .../Model/VectorAdapters.cs | 23 +- runtime/csharp/Prompty.Core/EndpointTrust.cs | 260 ++++++++++++++++++ .../Prompty.Core/PromptyLoadException.cs | 2 +- .../csharp/Prompty.Core/PromptyLoadOptions.cs | 7 + runtime/csharp/Prompty.Core/PromptyLoader.cs | 11 +- runtime/csharp/Prompty.Core/README.md | 23 ++ .../csharp/Prompty.Core/ReferenceResolver.cs | 66 ++++- 9 files changed, 588 insertions(+), 10 deletions(-) create mode 100644 runtime/csharp/Prompty.Core.Tests/EndpointTrustTests.cs create mode 100644 runtime/csharp/Prompty.Core/EndpointTrust.cs diff --git a/runtime/csharp/CHANGELOG.md b/runtime/csharp/CHANGELOG.md index 55a819d0a..646e90a6e 100644 --- a/runtime/csharp/CHANGELOG.md +++ b/runtime/csharp/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to the Prompty C# runtime will be documented here. +## [Unreleased] +### Security +- Loader enforces spec §2.5.1 endpoint trust: a `.prompty` whose `model.connection` would send ambient credentials to a file-chosen, untrusted `endpoint`/`tokenUrl` fails with `untrusted_endpoint`. New `PromptyLoadOptions.TrustedEndpoints` option and `PROMPTY_TRUSTED_ENDPOINTS` env var. + ## [2.0.0] — 2026-09-16 ### Added diff --git a/runtime/csharp/Prompty.Core.Tests/EndpointTrustTests.cs b/runtime/csharp/Prompty.Core.Tests/EndpointTrustTests.cs new file mode 100644 index 000000000..239d665e9 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/EndpointTrustTests.cs @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft. All rights reserved. +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// Tests for endpoint trust (spec §2.5.1). +public sealed class EndpointTrustTests : IDisposable +{ + private readonly string _tmp; + private readonly Dictionary _savedEnv = []; + + public EndpointTrustTests() + { + _tmp = Path.Combine(Path.GetTempPath(), $"prompty-trust-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tmp); + SetEnv(EndpointTrust.TrustedEndpointsEnv, null); + } + + public void Dispose() + { + foreach (var (key, value) in _savedEnv) + Environment.SetEnvironmentVariable(key, value); + try + { + Directory.Delete(_tmp, recursive: true); + } + catch (IOException) + { + } + } + + private void SetEnv(string key, string? value) + { + _savedEnv.TryAdd(key, Environment.GetEnvironmentVariable(key)); + Environment.SetEnvironmentVariable(key, value); + } + + private string Write(string connection) + { + var p = Path.Combine(_tmp, "t.prompty"); + File.WriteAllText(p, $"---\nname: t\nmodel:\n id: gpt-4\n connection:\n{connection}\n---\nHello\n"); + return p; + } + + // --- IsTrustedEndpoint --- + + [Theory] + [InlineData("https://api.openai.com/v1")] + [InlineData("https://API.OpenAI.com:443")] + [InlineData("https://myres.openai.azure.com/")] + [InlineData("https://a.b.services.ai.azure.com/api/projects/p")] + [InlineData("https://login.microsoftonline.com/tenant/oauth2/v2.0/token")] + public void BuiltinTrusted(string url) => Assert.True(EndpointTrust.IsTrustedEndpoint(url)); + + [Theory] + [InlineData("http://api.openai.com")] // scheme mismatch + [InlineData("https://api.openai.com:8443")] // port mismatch + [InlineData("https://openai.azure.com")] // wildcard never matches apex + [InlineData("https://api.openai.com.evil.com")] + [InlineData("https://evilopenai.azure.com")] + [InlineData("https://api.openai.com@evil.com")] // userinfo ignored; host is evil.com + [InlineData("http://localhost:11434")] + [InlineData("not a url")] + [InlineData("ftp://api.openai.com")] + [InlineData("")] + [InlineData("https://api.openai.com:99999")] // invalid port + [InlineData("https://api.openai.com:abc")] + [InlineData("https://[::1")] + public void Untrusted(string url) => Assert.False(EndpointTrust.IsTrustedEndpoint(url)); + + [Fact] + public void HostOptionAndWildcard() + { + Assert.True(EndpointTrust.IsTrustedEndpoint("http://localhost:11434/v1", ["http://LOCALHOST:11434"])); + Assert.True(EndpointTrust.IsTrustedEndpoint("https://x.contoso.com", ["https://*.contoso.com"])); + Assert.False(EndpointTrust.IsTrustedEndpoint("https://contoso.com", ["https://*.contoso.com"])); + Assert.True(EndpointTrust.IsTrustedEndpoint("http://[::1]:8080/x", ["http://[::1]:8080"])); + } + + [Fact] + public void EnvListAndStar() + { + SetEnv(EndpointTrust.TrustedEndpointsEnv, " https://a.example.com , http://localhost:8080"); + Assert.True(EndpointTrust.IsTrustedEndpoint("http://localhost:8080/x")); + Assert.False(EndpointTrust.IsTrustedEndpoint("http://localhost:8081")); + SetEnv(EndpointTrust.TrustedEndpointsEnv, "*"); + Assert.True(EndpointTrust.IsTrustedEndpoint("http://anything.example")); + } + + // --- Loader enforcement --- + + [Fact] + public void AmbientLiteralEndpointRejected() + { + var p = Write(" kind: foundry\n endpoint: http://127.0.0.1:9999"); + var ex = Assert.Throws(() => PromptyLoader.Load(p)); + Assert.Equal("untrusted_endpoint", ex.Kind); + Assert.Contains("127.0.0.1", ex.Message); + } + + [Fact] + public void EnvKeyLiteralEndpointRejected() + { + SetEnv("SOME_KEY", "secret"); + var p = Write(" kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}"); + var ex = Assert.Throws(() => PromptyLoader.Load(p)); + Assert.Contains("not trusted", ex.Message); + } + + [Fact] + public void OAuthTokenUrlChecked() + { + SetEnv("SECRET", "s"); + var p = Write( + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token"); + var ex = Assert.Throws(() => PromptyLoader.Load(p)); + Assert.Contains("tokenUrl", ex.Message); + } + + [Fact] + public void MismatchedCredentialFieldRejected() + { + var p = Write(" kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk"); + Assert.Contains("not trusted", Assert.Throws(() => PromptyLoader.Load(p)).Message); + p = Write(" kind: key\n endpoint: https://evil.example.com\n clientSecret: junk"); + Assert.Contains("not trusted", Assert.Throws(() => PromptyLoader.Load(p)).Message); + } + + [Fact] + public void LiteralKeyAllowed() + { + var agent = PromptyLoader.Load(Write(" kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine")); + Assert.Equal("http://127.0.0.1:9999", Assert.IsType(agent.Model!.Connection).Endpoint); + } + + [Fact] + public void EnvEndpointAllowed() + { + SetEnv("EP", "http://127.0.0.1:9999"); + var agent = PromptyLoader.Load(Write(" kind: foundry\n endpoint: ${env:EP}")); + Assert.Equal("http://127.0.0.1:9999", Assert.IsType(agent.Model!.Connection).Endpoint); + } + + [Fact] + public void EnvDefaultUsedRejected() + { + SetEnv("UNSET_EP", null); + var p = Write(" kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}"); + Assert.Contains("not trusted", Assert.Throws(() => PromptyLoader.Load(p)).Message); + } + + [Fact] + public void TrustedOptionAllows() + { + var p = Write(" kind: foundry\n endpoint: http://127.0.0.1:9999/api"); + var agent = PromptyLoader.Load(p, new PromptyLoadOptions { TrustedEndpoints = ["http://127.0.0.1:9999"] }); + Assert.Equal("http://127.0.0.1:9999/api", Assert.IsType(agent.Model!.Connection).Endpoint); + } + + [Fact] + public async Task AsyncHonorsOption() + { + var p = Write(" kind: foundry\n endpoint: http://127.0.0.1:9999"); + await Assert.ThrowsAsync(() => PromptyLoader.LoadAsync(p)); + var agent = await PromptyLoader.LoadAsync( + p, new PromptyLoadOptions { TrustedEndpoints = ["http://127.0.0.1:9999"] }); + Assert.Equal("http://127.0.0.1:9999", Assert.IsType(agent.Model!.Connection).Endpoint); + } + + [Fact] + public void FileModelReferenceChecked() + { + File.WriteAllText( + Path.Combine(_tmp, "model.json"), + """{"id": "gpt-4", "connection": {"kind": "foundry", "endpoint": "https://evil.example.com"}}"""); + var p = Path.Combine(_tmp, "t.prompty"); + File.WriteAllText(p, "---\nname: t\nmodel: ${file:model.json}\n---\nHi\n"); + Assert.Contains("not trusted", Assert.Throws(() => PromptyLoader.Load(p)).Message); + } + + [Fact] + public void FileConnectionReferenceChecked() + { + File.WriteAllText(Path.Combine(_tmp, "conn.yaml"), "kind: foundry\nendpoint: https://evil.example.com\n"); + var p = Path.Combine(_tmp, "t.prompty"); + File.WriteAllText(p, "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:conn.yaml}\n---\nHi\n"); + var ex = Assert.Throws(() => PromptyLoader.Load(p)); + Assert.Equal("untrusted_endpoint", ex.Kind); + } + + [Fact] + public void FrontmatterCannotSelfTrust() + { + var p = Path.Combine(_tmp, "t.prompty"); + File.WriteAllText( + p, + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n"); + Assert.Throws(() => PromptyLoader.Load(p)); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs index 4f82cd411..0e95c5c84 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs @@ -296,6 +296,19 @@ private static List LiveProviderMessages(Dictionary fl Environment.SetEnvironmentVariable("NONEXISTENT", null); } + // Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + if (input["env"] is not JsonObject envVars || !envVars.ContainsKey(EndpointTrust.TrustedEndpointsEnv)) + { + savedEnv.TryAdd(EndpointTrust.TrustedEndpointsEnv, Environment.GetEnvironmentVariable(EndpointTrust.TrustedEndpointsEnv)); + Environment.SetEnvironmentVariable(EndpointTrust.TrustedEndpointsEnv, null); + } + var loadOptions = new PromptyLoadOptions + { + TrustedEndpoints = input["trustedEndpoints"] is JsonArray trusted + ? trusted.Select(t => t?.GetValue() ?? string.Empty).ToList() + : [], + }; + try { // --- input-validation vectors: BOTH a top-level `inputs` map AND `frontmatter` --- @@ -325,7 +338,7 @@ private static List LiveProviderMessages(Dictionary fl Agent loaded; if (input["fixture"] is JsonValue fixtureVal) { - loaded = PromptyLoader.Load(Path.Combine(SpecFixtures, fixtureVal.GetValue())); + loaded = PromptyLoader.Load(Path.Combine(SpecFixtures, fixtureVal.GetValue()), loadOptions); } else if (input["frontmatter_raw"] is JsonValue rawVal) { @@ -334,7 +347,7 @@ private static List LiveProviderMessages(Dictionary fl { var p = Path.Combine(tmp, "vector.prompty"); File.WriteAllText(p, rawVal.GetValue()); - loaded = PromptyLoader.Load(p); + loaded = PromptyLoader.Load(p, loadOptions); } finally { @@ -343,7 +356,7 @@ private static List LiveProviderMessages(Dictionary fl } else if (input["frontmatter"] is JsonObject) { - loaded = MaterializeAndLoad(input); + loaded = MaterializeAndLoad(input, loadOptions); } else { @@ -436,7 +449,7 @@ private static Agent MakeAgentFromFrontmatter(JsonNode? frontmatter) /// .. are written relative to the agent directory so path-traversal vectors /// can place a target outside the allowed root; the loader must reject the escape. /// - private static Agent MaterializeAndLoad(JsonObject input) + private static Agent MaterializeAndLoad(JsonObject input, PromptyLoadOptions? options = null) { var tempBase = NewTempDir(); try @@ -464,7 +477,7 @@ private static Agent MaterializeAndLoad(JsonObject input) var frontmatter = input["frontmatter"]!.ToJsonString(); var agentPath = Path.Join(agentDir, "vector.prompty"); File.WriteAllText(agentPath, $"---\n{frontmatter}\n---\n"); - return PromptyLoader.Load(agentPath); + return PromptyLoader.Load(agentPath, options); } finally { diff --git a/runtime/csharp/Prompty.Core/EndpointTrust.cs b/runtime/csharp/Prompty.Core/EndpointTrust.cs new file mode 100644 index 000000000..4227c15ba --- /dev/null +++ b/runtime/csharp/Prompty.Core/EndpointTrust.cs @@ -0,0 +1,260 @@ +// Copyright (c) Microsoft. All rights reserved. +using System.Collections; +using System.Globalization; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Text.RegularExpressions; + +namespace Prompty.Core; + +/// +/// Endpoint trust — keeps ambient credentials on host-approved destinations (spec §2.5.1). +/// +/// A .prompty file is untrusted input. When its connection carries no file-supplied +/// credential, executors fall back to ambient credentials (DefaultAzureCredential, +/// *_API_KEY env vars). This class decides whether the file-chosen destination may +/// receive them. +/// +public static partial class EndpointTrust +{ + /// Env var holding comma-separated, host-approved trusted origins. + public const string TrustedEndpointsEnv = "PROMPTY_TRUSTED_ENDPOINTS"; + + /// First-party provider origins that are always trusted. + public static readonly IReadOnlyList BuiltinTrustedEndpoints = + [ + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com", + ]; + + private static readonly string[] DestinationKeys = ["endpoint", "tokenUrl"]; + + /// + /// Return true if 's origin matches a built-in, host-supplied, + /// or PROMPTY_TRUSTED_ENDPOINTS trusted origin. + /// + public static bool IsTrustedEndpoint(string url, IEnumerable? trustedEndpoints = null) + { + var entries = TrustedEntries(trustedEndpoints); + if (entries.Contains("*")) + return true; + + var target = Origin(url); + if (target is null) + return false; + var (scheme, host, port) = target.Value; + + foreach (var entry in entries) + { + var allowed = Origin(entry); + if (allowed is null || allowed.Value.Scheme != scheme || allowed.Value.Port != port) + continue; + var pattern = allowed.Value.Host; + if (pattern.StartsWith("*.", StringComparison.Ordinal)) + { + if (host.EndsWith(pattern[1..], StringComparison.Ordinal)) + return true; + } + else if (pattern == host) + { + return true; + } + } + + return false; + } + + /// + /// Throw an untrusted_endpoint if ambient + /// credentials could reach a file-chosen, untrusted destination. + /// + /// The RAW (unresolved) model.connection mapping. + /// Host-supplied trusted origins. + public static void CheckConnectionTrust(object? connection, IEnumerable? trustedEndpoints = null) + { + if (connection is not IDictionary conn || Get(conn, "kind") is "reference") + return; + if (FileSuppliedCredential(conn)) + return; + + foreach (var key in DestinationKeys) + { + var destination = FileControlledDestination(Get(conn, key)); + if (destination is not null && !IsTrustedEndpoint(destination, trustedEndpoints)) + { + throw new PromptyLoadException( + "untrusted_endpoint", + $"Connection {key} '{destination}' is not trusted to receive ambient credentials. " + + $"Set the {key} via ${{env:VAR}}, add its origin to TrustedEndpoints or " + + $"{TrustedEndpointsEnv}, or use 'kind: reference'."); + } + } + } + + private static List TrustedEntries(IEnumerable? trustedEndpoints) + { + var entries = new List(BuiltinTrustedEndpoints); + if (trustedEndpoints is not null) + entries.AddRange(trustedEndpoints); + entries.AddRange((Environment.GetEnvironmentVariable(TrustedEndpointsEnv) ?? string.Empty).Split(',')); + return entries + .Where(e => e is not null && e.Trim().Length > 0) + .Select(e => e!.Trim()) + .ToList(); + } + + private static object? Get(IDictionary dict, string key) => dict.Contains(key) ? dict[key] : null; + + private static bool IsReference(string value) => + value.StartsWith("${", StringComparison.Ordinal) && value.EndsWith('}'); + + private static bool FileSuppliedCredential(IDictionary connection) + { + // Only the credential field the declared kind actually consumes counts; any other kind is ambient. + var key = Get(connection, "kind") switch + { + "key" => "apiKey", + "oauth" => "clientSecret", + _ => null, + }; + if (key is null) + return false; + return Get(connection, key) is string value && value.Length > 0 && !IsReference(value); + } + + /// Return the effective destination if the file chose it, else null. + private static string? FileControlledDestination(object? value) + { + if (value is null) + return null; + if (value is not string str) + return Convert.ToString(value, CultureInfo.InvariantCulture); + if (IsReference(str)) + { + var inner = str[2..^1]; + var colon = inner.IndexOf(':'); + var protocol = colon < 0 ? inner : inner[..colon]; + var rest = colon < 0 ? string.Empty : inner[(colon + 1)..]; + if (protocol.Equals("env", StringComparison.OrdinalIgnoreCase)) + { + var varColon = rest.IndexOf(':'); + var varName = varColon < 0 ? rest : rest[..varColon]; + var defaultValue = varColon < 0 ? string.Empty : rest[(varColon + 1)..]; + if (Environment.GetEnvironmentVariable(varName) is not null) + return null; + return defaultValue.Length > 0 ? defaultValue : null; + } + } + + return str; + } + + // ----------------------------------------------------------------------- + // Origin parsing — mirrors Python's urllib.parse.urlsplit semantics so every + // runtime classifies the same URL identically (System.Uri differs on IDN, + // userinfo, and malformed ports). + // ----------------------------------------------------------------------- + + private readonly record struct OriginParts(string Scheme, string Host, int Port); + + [GeneratedRegex(@"^v[a-fA-F0-9]+\.[a-zA-Z0-9\._~\-!\$&'\(\)\*\+,;=:]+$")] + private static partial Regex IpvFutureRegex(); + + private static OriginParts? Origin(string url) + { + var s = new string(url.Trim().Where(c => c is not ('\t' or '\r' or '\n')).ToArray()); + + var scheme = string.Empty; + var rest = s; + var colon = s.IndexOf(':'); + if (colon > 0 && char.IsAsciiLetter(s[0]) + && s[..colon].All(c => char.IsAsciiLetterOrDigit(c) || c is '+' or '-' or '.')) + { + scheme = s[..colon].ToLowerInvariant(); + rest = s[(colon + 1)..]; + } + + if (scheme is not ("http" or "https")) + return null; + + var netloc = string.Empty; + if (rest.StartsWith("//", StringComparison.Ordinal)) + { + var after = rest[2..]; + var end = after.IndexOfAny(['/', '?', '#']); + netloc = end < 0 ? after : after[..end]; + } + + if (netloc.Contains('[') != netloc.Contains(']')) + return null; + if (!CheckNetloc(netloc)) + return null; + + var at = netloc.LastIndexOf('@'); + var hostinfo = at < 0 ? netloc : netloc[(at + 1)..]; + string hostname; + string port; + var openBracket = hostinfo.IndexOf('['); + if (openBracket >= 0) + { + var bracketed = hostinfo[(openBracket + 1)..]; + var close = bracketed.IndexOf(']'); + hostname = close < 0 ? bracketed : bracketed[..close]; + var afterClose = close < 0 ? string.Empty : bracketed[(close + 1)..]; + var portColon = afterClose.IndexOf(':'); + port = portColon < 0 ? string.Empty : afterClose[(portColon + 1)..]; + if (!IsValidBracketedHost(hostname)) + return null; + } + else + { + var portColon = hostinfo.IndexOf(':'); + hostname = portColon < 0 ? hostinfo : hostinfo[..portColon]; + port = portColon < 0 ? string.Empty : hostinfo[(portColon + 1)..]; + } + + hostname = hostname.ToLowerInvariant(); + if (hostname.Length == 0) + return null; + + int portNumber; + if (port.Length == 0) + { + portNumber = scheme == "http" ? 80 : 443; + } + else + { + if (!port.All(char.IsAsciiDigit)) + return null; + var trimmed = port.TrimStart('0'); + if (trimmed.Length > 5 || !int.TryParse(trimmed.Length == 0 ? "0" : trimmed, out portNumber) + || portNumber > 65535) + return null; + } + + return new OriginParts(scheme, hostname, portNumber); + } + + private static bool CheckNetloc(string netloc) + { + if (netloc.Length == 0 || netloc.All(char.IsAscii)) + return true; + var n = netloc.Replace("@", "").Replace(":", "").Replace("#", "").Replace("?", ""); + var normalized = n.Normalize(NormalizationForm.FormKC); + if (n == normalized) + return true; + return normalized.IndexOfAny(['/', '?', '#', '@', ':']) < 0; + } + + private static bool IsValidBracketedHost(string host) + { + if (host.StartsWith('v')) + return IpvFutureRegex().IsMatch(host); + return IPAddress.TryParse(host, out var ip) && ip.AddressFamily == AddressFamily.InterNetworkV6; + } +} diff --git a/runtime/csharp/Prompty.Core/PromptyLoadException.cs b/runtime/csharp/Prompty.Core/PromptyLoadException.cs index 23b0561f9..4192e584b 100644 --- a/runtime/csharp/Prompty.Core/PromptyLoadException.cs +++ b/runtime/csharp/Prompty.Core/PromptyLoadException.cs @@ -7,7 +7,7 @@ namespace Prompty.Core; /// /// The kind is drawn from the cross-runtime load-error taxonomy /// (env_var_not_set, file_reference, invalid_frontmatter, -/// invalid_template, ...), letting conformance adapters map a runtime failure to +/// invalid_template, untrusted_endpoint, ...), letting conformance adapters map a runtime failure to /// the spec's expectedError by exception type — never by message /// substring matching. /// diff --git a/runtime/csharp/Prompty.Core/PromptyLoadOptions.cs b/runtime/csharp/Prompty.Core/PromptyLoadOptions.cs index 0f269fdfd..acad8eac5 100644 --- a/runtime/csharp/Prompty.Core/PromptyLoadOptions.cs +++ b/runtime/csharp/Prompty.Core/PromptyLoadOptions.cs @@ -12,4 +12,11 @@ public sealed class PromptyLoadOptions /// The prompt file's directory is always allowed. /// public IEnumerable AllowedFileRoots { get; init; } = []; + + /// + /// Additional origins (e.g. http://localhost:11434, https://*.contoso.com) + /// trusted to receive ambient credentials (spec §2.5.1). Unioned with the built-in + /// provider origins and the PROMPTY_TRUSTED_ENDPOINTS env var; "*" trusts all. + /// + public IEnumerable TrustedEndpoints { get; init; } = []; } diff --git a/runtime/csharp/Prompty.Core/PromptyLoader.cs b/runtime/csharp/Prompty.Core/PromptyLoader.cs index 74732ddbc..973e2747a 100644 --- a/runtime/csharp/Prompty.Core/PromptyLoader.cs +++ b/runtime/csharp/Prompty.Core/PromptyLoader.cs @@ -4,7 +4,7 @@ namespace Prompty.Core; /// /// Loads .prompty files into typed Prompty instances. -/// Flow: read file → parse frontmatter/body → resolve ${env:}/${file:} refs → Prompty.Load(). +/// Flow: read file → parse frontmatter/body → check endpoint trust (§2.5.1) → resolve ${env:}/${file:} refs → Prompty.Load(). /// public static class PromptyLoader { @@ -91,7 +91,12 @@ private static Agent Build(string contents, string fullPath, PromptyLoadOptions? $"Template must be an object with format/parser, got string '{templateString}'."); } - // 2. Load via typed model with ${env:}/${file:} resolution + // 2. Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. + EndpointTrust.CheckConnectionTrust( + ReferenceResolver.RawConnection(data, fullPath, options?.AllowedFileRoots), + options?.TrustedEndpoints); + + // 3. Load via typed model with ${env:}/${file:} resolution var ctx = new LoadContext { PreProcess = ReferenceResolver.CreatePreProcess(fullPath, options?.AllowedFileRoots), @@ -99,7 +104,7 @@ private static Agent Build(string contents, string fullPath, PromptyLoadOptions? var agent = Agent.Load(data, ctx); - // 3. Attach source path in metadata + // 4. Attach source path in metadata agent.Metadata ??= new Dictionary(); agent.Metadata["__source_path"] = fullPath; diff --git a/runtime/csharp/Prompty.Core/README.md b/runtime/csharp/Prompty.Core/README.md index a8f4f2b23..81ce2a6c1 100644 --- a/runtime/csharp/Prompty.Core/README.md +++ b/runtime/csharp/Prompty.Core/README.md @@ -101,6 +101,29 @@ var agent = PromptyLoader.Load("chat.prompty"); var agent = await PromptyLoader.LoadAsync("chat.prompty"); ``` +#### Load options + +```csharp +var agent = PromptyLoader.Load("chat.prompty", new PromptyLoadOptions +{ + // Extra directories ${file:...} references may read from (the prompt's directory is always allowed). + AllowedFileRoots = ["/shared/config"], + // Extra origins trusted to receive ambient credentials (spec §2.5.1). + TrustedEndpoints = ["http://localhost:11434", "https://*.contoso.com"], +}); +``` + +**Endpoint trust.** A `.prompty` file is untrusted input, so the loader refuses +(`PromptyLoadException` with `Kind == "untrusted_endpoint"`) to load a file whose +`model.connection` would send ambient credentials (`DefaultAzureCredential` tokens, +`*_API_KEY` env vars) to an endpoint or `tokenUrl` the file chose. Built-in provider +origins (`https://api.openai.com`, `https://api.anthropic.com`, `https://*.openai.azure.com`, +`https://*.services.ai.azure.com`, `https://*.cognitiveservices.azure.com`, +`https://login.microsoftonline.com`) are always trusted. To allow another destination, +set the endpoint via `${env:VAR}`, add its origin to `TrustedEndpoints` or the +comma-separated `PROMPTY_TRUSTED_ENDPOINTS` env var (`*` trusts all), supply a literal +credential in the file, or use `kind: reference`. + ### Pipeline ```csharp diff --git a/runtime/csharp/Prompty.Core/ReferenceResolver.cs b/runtime/csharp/Prompty.Core/ReferenceResolver.cs index dd8a71ecf..d4519fdf2 100644 --- a/runtime/csharp/Prompty.Core/ReferenceResolver.cs +++ b/runtime/csharp/Prompty.Core/ReferenceResolver.cs @@ -33,6 +33,66 @@ public static class ReferenceResolver return data => ResolveReferences(data, parentDir, allowedRoots); } + /// + /// Return the unresolved model.connection from raw frontmatter, following + /// whole-value ${file:...} references for model and model.connection + /// with the same containment rules as reference resolution (spec §2.5.1). + /// + internal static object? RawConnection( + Dictionary data, + string promptyFilePath, + IEnumerable? allowedFileRoots = null) + { + var parentDir = Path.GetDirectoryName(GetCanonicalPath(promptyFilePath)) + ?? throw new ArgumentException($"Cannot determine parent directory of '{promptyFilePath}'"); + var allowedRoots = new[] { parentDir } + .Concat(allowedFileRoots ?? []) + .Select(GetCanonicalPath) + .ToArray(); + + object? Follow(object? value, string key) + { + if (value is string str && str.StartsWith(RefPrefix) && str.EndsWith(RefSuffix)) + { + var inner = str[RefPrefix.Length..^RefSuffix.Length]; + var colonIndex = inner.IndexOf(':'); + if (colonIndex >= 0 && inner[..colonIndex].Equals("file", StringComparison.OrdinalIgnoreCase)) + return LoadContainedFile(inner[(colonIndex + 1)..], parentDir, allowedRoots, key); + } + return value; + } + + var model = Follow(data.GetValueOrDefault("model"), "model"); + if (model is not System.Collections.IDictionary modelDict) + return null; + return Follow(modelDict.Contains("connection") ? modelDict["connection"] : null, "connection"); + } + + /// + /// Containment-check then load a ${file:} target; returns null if it does not exist. + /// + private static object? LoadContainedFile( + string relativePath, + string parentDir, + IReadOnlyCollection allowedRoots, + string key) + { + var fullPath = GetCanonicalPath(Path.IsPathRooted(relativePath) + ? Path.GetFullPath(relativePath) + : Path.GetFullPath(Path.Combine(parentDir, relativePath))); + if (!allowedRoots.Any(root => IsWithinRoot(fullPath, root))) + { + throw new PromptyLoadException( + "file_reference", + $"File reference '{relativePath}' resolves outside allowed roots (resolved to '{fullPath}', key: '{key}')."); + } + + if (!File.Exists(fullPath)) + return null; + + return LoadFileContent(File.ReadAllText(fullPath), fullPath); + } + /// /// Walks a dictionary and resolves any string values matching ${protocol:value} patterns. /// Only processes top-level string values in the given dictionary (recursive walking is @@ -132,7 +192,11 @@ private static object ResolveFileRef( $"File reference '{relativePath}' resolves outside allowed roots (resolved to '{fullPath}', key: '{key}')."); } - var content = File.ReadAllText(fullPath); + return LoadFileContent(File.ReadAllText(fullPath), fullPath); + } + + private static object LoadFileContent(string content, string fullPath) + { var ext = Path.GetExtension(fullPath).ToLowerInvariant(); return ext switch From 0137987809998a722085f880c6db8e41137c0c13 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 12:34:58 -0700 Subject: [PATCH 10/15] fix(security): reject backslash endpoints in trust check Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Model/VectorConformanceTests.cs | 206 +++++----- runtime/csharp/Prompty.Core/EndpointTrust.cs | 3 + .../Prompty.Core/Model/VectorConformance.cs | 14 + .../prompty/model/vector_conformance_test.go | 205 +++++----- .../vectorconformance/vector_conformance.go | 11 + .../com/microsoft/prompty/EndpointTrust.java | 4 + .../prompty/model/VectorConformance.java | 14 + .../prompty/model/VectorConformanceTests.java | 360 +++++++++--------- .../prompty/prompty/core/endpoint_trust.py | 3 + .../prompty/model/vector_conformance.py | 10 + .../tests/model/test_vector_conformance.py | 204 +++++----- .../prompty/tests/test_endpoint_trust.py | 1 + .../prompty/src/model/vector_conformance.rs | 14 + .../tests/model/vector_conformance_test.rs | 183 ++++----- .../Sources/PromptyModel/EndpointTrust.swift | 2 + .../PromptyModel/vector_conformance.swift | 17 + .../VectorConformanceTests.swift | 186 ++++----- .../core/src/model/vector-conformance.ts | 15 + .../tests/model/vector-conformance.test.ts | 6 + schema/model/conformance/vectors/load.tsp | 20 + .../tsp-output/.typra-generated/vectors.json | 32 ++ spec/spec.md | 3 +- 22 files changed, 896 insertions(+), 617 deletions(-) diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs index acc0f91c5..fdb644aee 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs @@ -2048,7 +2048,37 @@ public async Task Vector54LoadConformanceLoadUntrustedEndpointAmbientRejected() } [Fact] - public async Task Vector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() + public async Task Vector55LoadConformanceLoadUntrustedEndpointBackslashRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_backslash_rejected", + "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-backslash", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com\\@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() { string vectorJson = """ { @@ -2079,7 +2109,7 @@ public async Task Vector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected } [Fact] - public async Task Vector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected() + public async Task Vector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected() { string vectorJson = """ { @@ -2113,7 +2143,7 @@ public async Task Vector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected() } [Fact] - public async Task Vector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected() + public async Task Vector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected() { string vectorJson = """ { @@ -2146,7 +2176,7 @@ public async Task Vector57LoadConformanceLoadUntrustedEndpointFileConnectionReje } [Fact] - public async Task Vector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() + public async Task Vector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() { string vectorJson = """ { @@ -2177,7 +2207,7 @@ public async Task Vector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecre } [Fact] - public async Task Vector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() + public async Task Vector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() { string vectorJson = """ { @@ -2208,7 +2238,7 @@ public async Task Vector59LoadConformanceLoadUntrustedEndpointMismatchedCredenti } [Fact] - public async Task Vector60LoadConformanceLoadUntrustedEndpointNoKeyRejected() + public async Task Vector61LoadConformanceLoadUntrustedEndpointNoKeyRejected() { string vectorJson = """ { @@ -2238,7 +2268,7 @@ public async Task Vector60LoadConformanceLoadUntrustedEndpointNoKeyRejected() } [Fact] - public async Task Vector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() + public async Task Vector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() { string vectorJson = """ { @@ -2271,7 +2301,7 @@ public async Task Vector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejecte } [Fact] - public async Task Vector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected() + public async Task Vector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected() { string vectorJson = """ { @@ -2301,7 +2331,7 @@ public async Task Vector62LoadConformanceLoadUntrustedEndpointWildcardApexReject } [Fact] - public async Task Vector63MemoryConformanceOperateClearByCategoryAndAll() + public async Task Vector64MemoryConformanceOperateClearByCategoryAndAll() { string vectorJson = """ { @@ -2355,7 +2385,7 @@ public async Task Vector63MemoryConformanceOperateClearByCategoryAndAll() } [Fact] - public async Task Vector64MemoryConformanceOperateFormatCoreAndRecallResults() + public async Task Vector65MemoryConformanceOperateFormatCoreAndRecallResults() { string vectorJson = """ { @@ -2397,7 +2427,7 @@ public async Task Vector64MemoryConformanceOperateFormatCoreAndRecallResults() } [Fact] - public async Task Vector65MemoryConformanceOperateRecallEmptyQueryAndLimit() + public async Task Vector66MemoryConformanceOperateRecallEmptyQueryAndLimit() { string vectorJson = """ { @@ -2452,7 +2482,7 @@ public async Task Vector65MemoryConformanceOperateRecallEmptyQueryAndLimit() } [Fact] - public async Task Vector66MemoryConformanceOperateRecallRanksByWeightedScore() + public async Task Vector67MemoryConformanceOperateRecallRanksByWeightedScore() { string vectorJson = """ { @@ -2511,7 +2541,7 @@ public async Task Vector66MemoryConformanceOperateRecallRanksByWeightedScore() } [Fact] - public async Task Vector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() + public async Task Vector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() { string vectorJson = """ { @@ -2569,7 +2599,7 @@ public async Task Vector67MemoryConformanceOperateRecallScoreTiesPreserveInserti } [Fact] - public async Task Vector68MemoryConformanceOperateRecallTagsAndCoreBoost() + public async Task Vector69MemoryConformanceOperateRecallTagsAndCoreBoost() { string vectorJson = """ { @@ -2630,7 +2660,7 @@ public async Task Vector68MemoryConformanceOperateRecallTagsAndCoreBoost() } [Fact] - public async Task Vector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() + public async Task Vector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() { string vectorJson = """ { @@ -2691,7 +2721,7 @@ public async Task Vector69MemoryConformanceOperateRecallTokenizationPunctuationU } [Fact] - public async Task Vector70MemoryConformanceOperateRememberDedupsCoreByTags() + public async Task Vector71MemoryConformanceOperateRememberDedupsCoreByTags() { string vectorJson = """ { @@ -2755,7 +2785,7 @@ public async Task Vector70MemoryConformanceOperateRememberDedupsCoreByTags() } [Fact] - public async Task Vector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() + public async Task Vector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() { string vectorJson = """ { @@ -2811,7 +2841,7 @@ public async Task Vector71MemoryConformanceOperateRememberDedupsCoreMissingAndEm } [Fact] - public async Task Vector72MemoryConformanceOperateRememberEvictsArchivalFirst() + public async Task Vector73MemoryConformanceOperateRememberEvictsArchivalFirst() { string vectorJson = """ { @@ -2872,7 +2902,7 @@ public async Task Vector72MemoryConformanceOperateRememberEvictsArchivalFirst() } [Fact] - public async Task Vector73MemoryConformanceOperateSnapshotPortRoundTrip() + public async Task Vector74MemoryConformanceOperateSnapshotPortRoundTrip() { string vectorJson = """ { @@ -2912,7 +2942,7 @@ public async Task Vector73MemoryConformanceOperateSnapshotPortRoundTrip() } [Fact] - public async Task Vector74MemoryConformanceOperateUpdateIndexOutOfRangeError() + public async Task Vector75MemoryConformanceOperateUpdateIndexOutOfRangeError() { string vectorJson = """ { @@ -2946,7 +2976,7 @@ public async Task Vector74MemoryConformanceOperateUpdateIndexOutOfRangeError() } [Fact] - public async Task Vector75TurnConformanceReplayMaxIterations() + public async Task Vector76TurnConformanceReplayMaxIterations() { string vectorJson = """ { @@ -2983,7 +3013,7 @@ public async Task Vector75TurnConformanceReplayMaxIterations() } [Fact] - public async Task Vector76TurnConformanceReplayNoTool() + public async Task Vector77TurnConformanceReplayNoTool() { string vectorJson = """ { @@ -3016,7 +3046,7 @@ public async Task Vector76TurnConformanceReplayNoTool() } [Fact] - public async Task Vector77TurnConformanceReplayPermissionDenied() + public async Task Vector78TurnConformanceReplayPermissionDenied() { string vectorJson = """ { @@ -3052,7 +3082,7 @@ public async Task Vector77TurnConformanceReplayPermissionDenied() } [Fact] - public async Task Vector78TurnConformanceReplayToolFailure() + public async Task Vector79TurnConformanceReplayToolFailure() { string vectorJson = """ { @@ -3090,7 +3120,7 @@ public async Task Vector78TurnConformanceReplayToolFailure() } [Fact] - public async Task Vector79TurnConformanceReplayToolSuccess() + public async Task Vector80TurnConformanceReplayToolSuccess() { string vectorJson = """ { @@ -3128,7 +3158,7 @@ public async Task Vector79TurnConformanceReplayToolSuccess() } [Fact] - public async Task Vector80TurnConformanceRunAssistantToolCallsMetadata() + public async Task Vector81TurnConformanceRunAssistantToolCallsMetadata() { string vectorJson = """ { @@ -3282,7 +3312,7 @@ public async Task Vector80TurnConformanceRunAssistantToolCallsMetadata() } [Fact] - public async Task Vector81TurnConformanceRunAsyncToolFunction() + public async Task Vector82TurnConformanceRunAsyncToolFunction() { string vectorJson = """ { @@ -3394,7 +3424,7 @@ public async Task Vector81TurnConformanceRunAsyncToolFunction() } [Fact] - public async Task Vector82TurnConformanceRunBindingsInjected() + public async Task Vector83TurnConformanceRunBindingsInjected() { string vectorJson = """ { @@ -3526,7 +3556,7 @@ public async Task Vector82TurnConformanceRunBindingsInjected() } [Fact] - public async Task Vector83TurnConformanceRunCancellationBeforeLlm() + public async Task Vector84TurnConformanceRunCancellationBeforeLlm() { string vectorJson = """ { @@ -3589,7 +3619,7 @@ public async Task Vector83TurnConformanceRunCancellationBeforeLlm() } [Fact] - public async Task Vector84TurnConformanceRunCancellationBetweenIterations() + public async Task Vector85TurnConformanceRunCancellationBetweenIterations() { string vectorJson = """ { @@ -3721,7 +3751,7 @@ public async Task Vector84TurnConformanceRunCancellationBetweenIterations() } [Fact] - public async Task Vector85TurnConformanceRunCancellationBetweenTools() + public async Task Vector86TurnConformanceRunCancellationBetweenTools() { string vectorJson = """ { @@ -3862,7 +3892,7 @@ public async Task Vector85TurnConformanceRunCancellationBetweenTools() } [Fact] - public async Task Vector86TurnConformanceRunContextNoTrimWhenFits() + public async Task Vector87TurnConformanceRunContextNoTrimWhenFits() { string vectorJson = """ { @@ -3935,7 +3965,7 @@ public async Task Vector86TurnConformanceRunContextNoTrimWhenFits() } [Fact] - public async Task Vector87TurnConformanceRunContextPreservesSystemMessages() + public async Task Vector88TurnConformanceRunContextPreservesSystemMessages() { string vectorJson = """ { @@ -4044,7 +4074,7 @@ public async Task Vector87TurnConformanceRunContextPreservesSystemMessages() } [Fact] - public async Task Vector88TurnConformanceRunContextTrimBasic() + public async Task Vector89TurnConformanceRunContextTrimBasic() { string vectorJson = """ { @@ -4204,7 +4234,7 @@ public async Task Vector88TurnConformanceRunContextTrimBasic() } [Fact] - public async Task Vector89TurnConformanceRunEmptyToolResult() + public async Task Vector90TurnConformanceRunEmptyToolResult() { string vectorJson = """ { @@ -4321,7 +4351,7 @@ public async Task Vector89TurnConformanceRunEmptyToolResult() } [Fact] - public async Task Vector90TurnConformanceRunEventsBasicToolLoop() + public async Task Vector91TurnConformanceRunEventsBasicToolLoop() { string vectorJson = """ { @@ -4469,7 +4499,7 @@ public async Task Vector90TurnConformanceRunEventsBasicToolLoop() } [Fact] - public async Task Vector91TurnConformanceRunEventsErrorLogged() + public async Task Vector92TurnConformanceRunEventsErrorLogged() { string vectorJson = """ { @@ -4597,7 +4627,7 @@ public async Task Vector91TurnConformanceRunEventsErrorLogged() } [Fact] - public async Task Vector92TurnConformanceRunEventsNoTools() + public async Task Vector93TurnConformanceRunEventsNoTools() { string vectorJson = """ { @@ -4677,7 +4707,7 @@ public async Task Vector92TurnConformanceRunEventsNoTools() } [Fact] - public async Task Vector93TurnConformanceRunGuardrailAllPass() + public async Task Vector94TurnConformanceRunGuardrailAllPass() { string vectorJson = """ { @@ -4802,7 +4832,7 @@ public async Task Vector93TurnConformanceRunGuardrailAllPass() } [Fact] - public async Task Vector94TurnConformanceRunGuardrailInputDeny() + public async Task Vector95TurnConformanceRunGuardrailInputDeny() { string vectorJson = """ { @@ -4860,7 +4890,7 @@ public async Task Vector94TurnConformanceRunGuardrailInputDeny() } [Fact] - public async Task Vector95TurnConformanceRunGuardrailOutputDeny() + public async Task Vector96TurnConformanceRunGuardrailOutputDeny() { string vectorJson = """ { @@ -4940,7 +4970,7 @@ public async Task Vector95TurnConformanceRunGuardrailOutputDeny() } [Fact] - public async Task Vector96TurnConformanceRunGuardrailToolDeny() + public async Task Vector97TurnConformanceRunGuardrailToolDeny() { string vectorJson = """ { @@ -5096,7 +5126,7 @@ public async Task Vector96TurnConformanceRunGuardrailToolDeny() } [Fact] - public async Task Vector97TurnConformanceRunMaxIterationsExceeded() + public async Task Vector98TurnConformanceRunMaxIterationsExceeded() { string vectorJson = """ { @@ -5608,7 +5638,7 @@ public async Task Vector97TurnConformanceRunMaxIterationsExceeded() } [Fact] - public async Task Vector98TurnConformanceRunMultiTurnToolCalls() + public async Task Vector99TurnConformanceRunMultiTurnToolCalls() { string vectorJson = """ { @@ -5787,7 +5817,7 @@ public async Task Vector98TurnConformanceRunMultiTurnToolCalls() } [Fact] - public async Task Vector99TurnConformanceRunMultipleToolCallsSingleTurn() + public async Task Vector100TurnConformanceRunMultipleToolCallsSingleTurn() { string vectorJson = """ { @@ -5970,7 +6000,7 @@ public async Task Vector99TurnConformanceRunMultipleToolCallsSingleTurn() } [Fact] - public async Task Vector100TurnConformanceRunNoToolCalls() + public async Task Vector101TurnConformanceRunNoToolCalls() { string vectorJson = """ { @@ -6040,7 +6070,7 @@ public async Task Vector100TurnConformanceRunNoToolCalls() } [Fact] - public async Task Vector101TurnConformanceRunParallelToolsBasic() + public async Task Vector102TurnConformanceRunParallelToolsBasic() { string vectorJson = """ { @@ -6215,7 +6245,7 @@ public async Task Vector101TurnConformanceRunParallelToolsBasic() } [Fact] - public async Task Vector102TurnConformanceRunParallelToolsWithGuardrailDeny() + public async Task Vector103TurnConformanceRunParallelToolsWithGuardrailDeny() { string vectorJson = """ { @@ -6408,7 +6438,7 @@ public async Task Vector102TurnConformanceRunParallelToolsWithGuardrailDeny() } [Fact] - public async Task Vector103TurnConformanceRunParallelToolsWithToolError() + public async Task Vector104TurnConformanceRunParallelToolsWithToolError() { string vectorJson = """ { @@ -6583,7 +6613,7 @@ public async Task Vector103TurnConformanceRunParallelToolsWithToolError() } [Fact] - public async Task Vector104TurnConformanceRunSingleToolCall() + public async Task Vector105TurnConformanceRunSingleToolCall() { string vectorJson = """ { @@ -6732,7 +6762,7 @@ public async Task Vector104TurnConformanceRunSingleToolCall() } [Fact] - public async Task Vector105TurnConformanceRunSteeringInjectMessage() + public async Task Vector106TurnConformanceRunSteeringInjectMessage() { string vectorJson = """ { @@ -6942,7 +6972,7 @@ public async Task Vector105TurnConformanceRunSteeringInjectMessage() } [Fact] - public async Task Vector106TurnConformanceRunSteeringMultipleMessages() + public async Task Vector107TurnConformanceRunSteeringMultipleMessages() { string vectorJson = """ { @@ -7114,7 +7144,7 @@ public async Task Vector106TurnConformanceRunSteeringMultipleMessages() } [Fact] - public async Task Vector107TurnConformanceRunToolNotRegisteredError() + public async Task Vector108TurnConformanceRunToolNotRegisteredError() { string vectorJson = """ { @@ -7200,7 +7230,7 @@ public async Task Vector107TurnConformanceRunToolNotRegisteredError() } [Fact] - public async Task Vector108TurnConformanceRunToolResultMessageFormat() + public async Task Vector109TurnConformanceRunToolResultMessageFormat() { string vectorJson = """ { @@ -7323,7 +7353,7 @@ public async Task Vector108TurnConformanceRunToolResultMessageFormat() } [Fact] - public async Task Vector109TurnConformanceRunTurnCancelBeforeContext() + public async Task Vector110TurnConformanceRunTurnCancelBeforeContext() { string vectorJson = """ { @@ -7357,7 +7387,7 @@ public async Task Vector109TurnConformanceRunTurnCancelBeforeContext() } [Fact] - public async Task Vector110TurnConformanceRunTurnDelegatedProviderState() + public async Task Vector111TurnConformanceRunTurnDelegatedProviderState() { string vectorJson = """ { @@ -7426,7 +7456,7 @@ public async Task Vector110TurnConformanceRunTurnDelegatedProviderState() } [Fact] - public async Task Vector111TurnConformanceRunTurnFinalOutput() + public async Task Vector112TurnConformanceRunTurnFinalOutput() { string vectorJson = """ { @@ -7473,7 +7503,7 @@ public async Task Vector111TurnConformanceRunTurnFinalOutput() } [Fact] - public async Task Vector112TurnConformanceRunTurnMemoryContextIsModelVisible() + public async Task Vector113TurnConformanceRunTurnMemoryContextIsModelVisible() { string vectorJson = """ { @@ -7537,7 +7567,7 @@ public async Task Vector112TurnConformanceRunTurnMemoryContextIsModelVisible() } [Fact] - public async Task Vector113TurnConformanceRunTurnOrderedToolRound() + public async Task Vector114TurnConformanceRunTurnOrderedToolRound() { string vectorJson = """ { @@ -7630,7 +7660,7 @@ public async Task Vector113TurnConformanceRunTurnOrderedToolRound() } [Fact] - public async Task Vector114TurnConformanceRunTurnPermissionDenialIsModelVisible() + public async Task Vector115TurnConformanceRunTurnPermissionDenialIsModelVisible() { string vectorJson = """ { @@ -7683,7 +7713,7 @@ public async Task Vector114TurnConformanceRunTurnPermissionDenialIsModelVisible( } [Fact] - public async Task Vector115WireConformanceToRequestAnthropicImageFormat() + public async Task Vector116WireConformanceToRequestAnthropicImageFormat() { string vectorJson = """ { @@ -7755,7 +7785,7 @@ public async Task Vector115WireConformanceToRequestAnthropicImageFormat() } [Fact] - public async Task Vector116WireConformanceToRequestAnthropicMaxTokensRequired() + public async Task Vector117WireConformanceToRequestAnthropicMaxTokensRequired() { string vectorJson = """ { @@ -7812,7 +7842,7 @@ public async Task Vector116WireConformanceToRequestAnthropicMaxTokensRequired() } [Fact] - public async Task Vector117WireConformanceToRequestAnthropicOptions() + public async Task Vector118WireConformanceToRequestAnthropicOptions() { string vectorJson = """ { @@ -7883,7 +7913,7 @@ public async Task Vector117WireConformanceToRequestAnthropicOptions() } [Fact] - public async Task Vector118WireConformanceToRequestAnthropicSystemSeparate() + public async Task Vector119WireConformanceToRequestAnthropicSystemSeparate() { string vectorJson = """ { @@ -7952,7 +7982,7 @@ public async Task Vector118WireConformanceToRequestAnthropicSystemSeparate() } [Fact] - public async Task Vector119WireConformanceToRequestAnthropicToolWire() + public async Task Vector120WireConformanceToRequestAnthropicToolWire() { string vectorJson = """ { @@ -8039,7 +8069,7 @@ public async Task Vector119WireConformanceToRequestAnthropicToolWire() } [Fact] - public async Task Vector120WireConformanceToRequestChatAudioMp3() + public async Task Vector121WireConformanceToRequestChatAudioMp3() { string vectorJson = """ { @@ -8099,7 +8129,7 @@ public async Task Vector120WireConformanceToRequestChatAudioMp3() } [Fact] - public async Task Vector121WireConformanceToRequestChatAudioPart() + public async Task Vector122WireConformanceToRequestChatAudioPart() { string vectorJson = """ { @@ -8159,7 +8189,7 @@ public async Task Vector121WireConformanceToRequestChatAudioPart() } [Fact] - public async Task Vector122WireConformanceToRequestChatImageBase64() + public async Task Vector123WireConformanceToRequestChatImageBase64() { string vectorJson = """ { @@ -8225,7 +8255,7 @@ public async Task Vector122WireConformanceToRequestChatImageBase64() } [Fact] - public async Task Vector123WireConformanceToRequestChatImagePart() + public async Task Vector124WireConformanceToRequestChatImagePart() { string vectorJson = """ { @@ -8291,7 +8321,7 @@ public async Task Vector123WireConformanceToRequestChatImagePart() } [Fact] - public async Task Vector124WireConformanceToRequestChatMultipartContent() + public async Task Vector125WireConformanceToRequestChatMultipartContent() { string vectorJson = """ { @@ -8357,7 +8387,7 @@ public async Task Vector124WireConformanceToRequestChatMultipartContent() } [Fact] - public async Task Vector125WireConformanceToRequestChatSimple() + public async Task Vector126WireConformanceToRequestChatSimple() { string vectorJson = """ { @@ -8421,7 +8451,7 @@ public async Task Vector125WireConformanceToRequestChatSimple() } [Fact] - public async Task Vector126WireConformanceToRequestChatSingleTextOptimized() + public async Task Vector127WireConformanceToRequestChatSingleTextOptimized() { string vectorJson = """ { @@ -8472,7 +8502,7 @@ public async Task Vector126WireConformanceToRequestChatSingleTextOptimized() } [Fact] - public async Task Vector127WireConformanceToRequestChatWithOptions() + public async Task Vector128WireConformanceToRequestChatWithOptions() { string vectorJson = """ { @@ -8542,7 +8572,7 @@ public async Task Vector127WireConformanceToRequestChatWithOptions() } [Fact] - public async Task Vector128WireConformanceToRequestEmbeddingWire() + public async Task Vector129WireConformanceToRequestEmbeddingWire() { string vectorJson = """ { @@ -8588,7 +8618,7 @@ public async Task Vector128WireConformanceToRequestEmbeddingWire() } [Fact] - public async Task Vector129WireConformanceToRequestImageWire() + public async Task Vector130WireConformanceToRequestImageWire() { string vectorJson = """ { @@ -8634,7 +8664,7 @@ public async Task Vector129WireConformanceToRequestImageWire() } [Fact] - public async Task Vector130WireConformanceToRequestKindToJsonTypeMapping() + public async Task Vector131WireConformanceToRequestKindToJsonTypeMapping() { string vectorJson = """ { @@ -8755,7 +8785,7 @@ public async Task Vector130WireConformanceToRequestKindToJsonTypeMapping() } [Fact] - public async Task Vector131WireConformanceToRequestOptionsAdditionalProperties() + public async Task Vector132WireConformanceToRequestOptionsAdditionalProperties() { string vectorJson = """ { @@ -8811,7 +8841,7 @@ public async Task Vector131WireConformanceToRequestOptionsAdditionalProperties() } [Fact] - public async Task Vector132WireConformanceToRequestOptionsMaxCompletionTokens() + public async Task Vector133WireConformanceToRequestOptionsMaxCompletionTokens() { string vectorJson = """ { @@ -8865,7 +8895,7 @@ public async Task Vector132WireConformanceToRequestOptionsMaxCompletionTokens() } [Fact] - public async Task Vector133WireConformanceToRequestOptionsStopSequences() + public async Task Vector134WireConformanceToRequestOptionsStopSequences() { string vectorJson = """ { @@ -8925,7 +8955,7 @@ public async Task Vector133WireConformanceToRequestOptionsStopSequences() } [Fact] - public async Task Vector134WireConformanceToRequestResponsesSimple() + public async Task Vector135WireConformanceToRequestResponsesSimple() { string vectorJson = """ { @@ -8986,7 +9016,7 @@ public async Task Vector134WireConformanceToRequestResponsesSimple() } [Fact] - public async Task Vector135WireConformanceToRequestResponsesStructuredOutput() + public async Task Vector136WireConformanceToRequestResponsesStructuredOutput() { string vectorJson = """ { @@ -9075,7 +9105,7 @@ public async Task Vector135WireConformanceToRequestResponsesStructuredOutput() } [Fact] - public async Task Vector136WireConformanceToRequestResponsesWithTools() + public async Task Vector137WireConformanceToRequestResponsesWithTools() { string vectorJson = """ { @@ -9157,7 +9187,7 @@ public async Task Vector136WireConformanceToRequestResponsesWithTools() } [Fact] - public async Task Vector137WireConformanceToRequestStructuredOutput() + public async Task Vector138WireConformanceToRequestStructuredOutput() { string vectorJson = """ { @@ -9242,7 +9272,7 @@ public async Task Vector137WireConformanceToRequestStructuredOutput() } [Fact] - public async Task Vector138WireConformanceToRequestStructuredOutputNestedOptional() + public async Task Vector139WireConformanceToRequestStructuredOutputNestedOptional() { string vectorJson = """ { @@ -9346,7 +9376,7 @@ public async Task Vector138WireConformanceToRequestStructuredOutputNestedOptiona } [Fact] - public async Task Vector139WireConformanceToRequestToolsBindingsStripped() + public async Task Vector140WireConformanceToRequestToolsBindingsStripped() { string vectorJson = """ { @@ -9440,7 +9470,7 @@ public async Task Vector139WireConformanceToRequestToolsBindingsStripped() } [Fact] - public async Task Vector140WireConformanceToRequestToolsFunctionWire() + public async Task Vector141WireConformanceToRequestToolsFunctionWire() { string vectorJson = """ { @@ -9524,7 +9554,7 @@ public async Task Vector140WireConformanceToRequestToolsFunctionWire() } [Fact] - public async Task Vector141WireConformanceToRequestToolsNullWhenEmpty() + public async Task Vector142WireConformanceToRequestToolsNullWhenEmpty() { string vectorJson = """ { @@ -9575,7 +9605,7 @@ public async Task Vector141WireConformanceToRequestToolsNullWhenEmpty() } [Fact] - public async Task Vector142WireConformanceToRequestToolsStrictMode() + public async Task Vector143WireConformanceToRequestToolsStrictMode() { string vectorJson = """ { diff --git a/runtime/csharp/Prompty.Core/EndpointTrust.cs b/runtime/csharp/Prompty.Core/EndpointTrust.cs index 4227c15ba..59bd2517a 100644 --- a/runtime/csharp/Prompty.Core/EndpointTrust.cs +++ b/runtime/csharp/Prompty.Core/EndpointTrust.cs @@ -167,6 +167,9 @@ private static bool FileSuppliedCredential(IDictionary connection) private static OriginParts? Origin(string url) { + // HTTP clients treat '\' as '/', so parsers disagree on the host; reject outright. + if (url.Contains('\\')) + return null; var s = new string(url.Trim().Where(c => c is not ('\t' or '\r' or '\n')).ToArray()); var scheme = string.Empty; diff --git a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs index 8d3efa5f1..eb38b13eb 100644 --- a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs +++ b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs @@ -443,6 +443,20 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea } if (caught is null) throw new SeamConformanceException("untrusted_endpoint_ambient_rejected: expected an error"); } + { + // vector: untrusted_endpoint_backslash_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_backslash_rejected: expected an error"); + } { // vector: untrusted_endpoint_env_default_rejected var source = JsonSerializer.Deserialize("null")!; diff --git a/runtime/go/prompty/model/vector_conformance_test.go b/runtime/go/prompty/model/vector_conformance_test.go index 1f9b66c12..b95316b87 100644 --- a/runtime/go/prompty/model/vector_conformance_test.go +++ b/runtime/go/prompty/model/vector_conformance_test.go @@ -1974,7 +1974,36 @@ func TestVector54LoadConformanceLoadUntrustedEndpointAmbientRejected(t *testing. vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testing.T) { +func TestVector55LoadConformanceLoadUntrustedEndpointBackslashRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_backslash_rejected", + "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-backslash", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com\\@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_env_default_rejected", "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", @@ -2004,7 +2033,7 @@ func TestVector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testi vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T) { +func TestVector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_env_key_rejected", "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", @@ -2037,7 +2066,7 @@ func TestVector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *testing.T) { +func TestVector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_file_connection_rejected", "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", @@ -2069,7 +2098,7 @@ func TestVector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected(t *testing.T) { +func TestVector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_key_with_client_secret_rejected", "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", @@ -2099,7 +2128,7 @@ func TestVector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected(t *testing.T) { +func TestVector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_mismatched_credential_rejected", "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", @@ -2129,7 +2158,7 @@ func TestVector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejecte vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector60LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) { +func TestVector61LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_no_key_rejected", "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", @@ -2158,7 +2187,7 @@ func TestVector60LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *testing.T) { +func TestVector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_suffix_spoof_rejected", "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", @@ -2190,7 +2219,7 @@ func TestVector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *test vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *testing.T) { +func TestVector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_wildcard_apex_rejected", "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", @@ -2219,7 +2248,7 @@ func TestVector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *tes vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector63MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { +func TestVector64MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorJSON := `{ "name": "clear_by_category_and_all", "description": "Clear removes only the requested category, or all memories when no category is supplied.", @@ -2272,7 +2301,7 @@ func TestVector63MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector64MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { +func TestVector65MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { vectorJSON := `{ "name": "format_core_and_recall_results", "description": "Formatting injects only core memories into system prompts and formats recall results with category and tags.", @@ -2313,7 +2342,7 @@ func TestVector64MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector65MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { +func TestVector66MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { vectorJSON := `{ "name": "recall_empty_query_and_limit", "description": "Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.", @@ -2367,7 +2396,7 @@ func TestVector65MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector66MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { +func TestVector67MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { vectorJSON := `{ "name": "recall_ranks_by_weighted_score", "description": "Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.", @@ -2425,7 +2454,7 @@ func TestVector66MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { +func TestVector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { vectorJSON := `{ "name": "recall_score_ties_preserve_insertion_order", "description": "Recall preserves insertion order for memories with identical scores.", @@ -2482,7 +2511,7 @@ func TestVector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector68MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { +func TestVector69MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorJSON := `{ "name": "recall_tags_and_core_boost", "description": "Recall gives tag matches higher weight than content matches and boosts matching core memories.", @@ -2542,7 +2571,7 @@ func TestVector68MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { +func TestVector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { vectorJSON := `{ "name": "recall_tokenization_punctuation_unicode_and_dedup", "description": "Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.", @@ -2602,7 +2631,7 @@ func TestVector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAnd vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector70MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { +func TestVector71MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_by_tags", "description": "Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.", @@ -2665,7 +2694,7 @@ func TestVector70MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { +func TestVector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_missing_and_empty_tags", "description": "Remember treats missing and empty tag lists as equivalent for core-memory deduplication.", @@ -2720,7 +2749,7 @@ func TestVector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector72MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { +func TestVector73MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { vectorJSON := `{ "name": "remember_evicts_archival_first", "description": "Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.", @@ -2780,7 +2809,7 @@ func TestVector72MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing. vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector73MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { +func TestVector74MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorJSON := `{ "name": "snapshot_port_round_trip", "description": "A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.", @@ -2819,7 +2848,7 @@ func TestVector73MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector74MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { +func TestVector75MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { vectorJSON := `{ "name": "update_index_out_of_range_error", "description": "Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.", @@ -2852,7 +2881,7 @@ func TestVector74MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector75TurnConformanceReplayMaxIterations(t *testing.T) { +func TestVector76TurnConformanceReplayMaxIterations(t *testing.T) { vectorJSON := `{ "name": "max_iterations", "stage": "replay", @@ -2888,7 +2917,7 @@ func TestVector75TurnConformanceReplayMaxIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector76TurnConformanceReplayNoTool(t *testing.T) { +func TestVector77TurnConformanceReplayNoTool(t *testing.T) { vectorJSON := `{ "name": "no_tool", "stage": "replay", @@ -2920,7 +2949,7 @@ func TestVector76TurnConformanceReplayNoTool(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector77TurnConformanceReplayPermissionDenied(t *testing.T) { +func TestVector78TurnConformanceReplayPermissionDenied(t *testing.T) { vectorJSON := `{ "name": "permission_denied", "stage": "replay", @@ -2955,7 +2984,7 @@ func TestVector77TurnConformanceReplayPermissionDenied(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector78TurnConformanceReplayToolFailure(t *testing.T) { +func TestVector79TurnConformanceReplayToolFailure(t *testing.T) { vectorJSON := `{ "name": "tool_failure", "stage": "replay", @@ -2992,7 +3021,7 @@ func TestVector78TurnConformanceReplayToolFailure(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector79TurnConformanceReplayToolSuccess(t *testing.T) { +func TestVector80TurnConformanceReplayToolSuccess(t *testing.T) { vectorJSON := `{ "name": "tool_success", "stage": "replay", @@ -3029,7 +3058,7 @@ func TestVector79TurnConformanceReplayToolSuccess(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector80TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { +func TestVector81TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorJSON := `{ "name": "assistant_tool_calls_metadata", "description": "When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.", @@ -3182,7 +3211,7 @@ func TestVector80TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector81TurnConformanceRunAsyncToolFunction(t *testing.T) { +func TestVector82TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorJSON := `{ "name": "async_tool_function", "description": "Agent loop correctly handles async tool functions", @@ -3293,7 +3322,7 @@ func TestVector81TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector82TurnConformanceRunBindingsInjected(t *testing.T) { +func TestVector83TurnConformanceRunBindingsInjected(t *testing.T) { vectorJSON := `{ "name": "bindings_injected", "description": "Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.", @@ -3424,7 +3453,7 @@ func TestVector82TurnConformanceRunBindingsInjected(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector83TurnConformanceRunCancellationBeforeLlm(t *testing.T) { +func TestVector84TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorJSON := `{ "name": "cancellation_before_llm", "description": "§13.2 Cancellation — Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.", @@ -3486,7 +3515,7 @@ func TestVector83TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector84TurnConformanceRunCancellationBetweenIterations(t *testing.T) { +func TestVector85TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_iterations", "description": "§13.2 Cancellation — Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.", @@ -3617,7 +3646,7 @@ func TestVector84TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector85TurnConformanceRunCancellationBetweenTools(t *testing.T) { +func TestVector86TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_tools", "description": "§13.2 Cancellation — LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.", @@ -3757,7 +3786,7 @@ func TestVector85TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector86TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { +func TestVector87TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorJSON := `{ "name": "context_no_trim_when_fits", "description": "§13.3 Context Window — Messages fit within the context budget. No trimming occurs and all messages are preserved.", @@ -3829,7 +3858,7 @@ func TestVector86TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector87TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { +func TestVector88TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { vectorJSON := `{ "name": "context_preserves_system_messages", "description": "§13.3 Context Window — Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.", @@ -3937,7 +3966,7 @@ func TestVector87TurnConformanceRunContextPreservesSystemMessages(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector88TurnConformanceRunContextTrimBasic(t *testing.T) { +func TestVector89TurnConformanceRunContextTrimBasic(t *testing.T) { vectorJSON := `{ "name": "context_trim_basic", "description": "§13.3 Context Window — Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.", @@ -4096,7 +4125,7 @@ func TestVector88TurnConformanceRunContextTrimBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector89TurnConformanceRunEmptyToolResult(t *testing.T) { +func TestVector90TurnConformanceRunEmptyToolResult(t *testing.T) { vectorJSON := `{ "name": "empty_tool_result", "description": "Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content — it must not be skipped or filtered out.", @@ -4212,7 +4241,7 @@ func TestVector89TurnConformanceRunEmptyToolResult(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector90TurnConformanceRunEventsBasicToolLoop(t *testing.T) { +func TestVector91TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorJSON := `{ "name": "events_basic_tool_loop", "description": "§13.1 Events — A 2-turn tool-call loop emits the full event lifecycle: status → tool_call_start → tool_result → messages_updated → done.", @@ -4359,7 +4388,7 @@ func TestVector90TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector91TurnConformanceRunEventsErrorLogged(t *testing.T) { +func TestVector92TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorJSON := `{ "name": "events_error_logged", "description": "§13.1 Events — A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.", @@ -4486,7 +4515,7 @@ func TestVector91TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector92TurnConformanceRunEventsNoTools(t *testing.T) { +func TestVector93TurnConformanceRunEventsNoTools(t *testing.T) { vectorJSON := `{ "name": "events_no_tools", "description": "§13.1 Events — Single-turn completion with no tool calls. Only the done event is emitted.", @@ -4565,7 +4594,7 @@ func TestVector92TurnConformanceRunEventsNoTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector93TurnConformanceRunGuardrailAllPass(t *testing.T) { +func TestVector94TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorJSON := `{ "name": "guardrail_all_pass", "description": "§13.4 Guardrails — All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.", @@ -4689,7 +4718,7 @@ func TestVector93TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector94TurnConformanceRunGuardrailInputDeny(t *testing.T) { +func TestVector95TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_input_deny", "description": "§13.4 Guardrails — Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.", @@ -4746,7 +4775,7 @@ func TestVector94TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector95TurnConformanceRunGuardrailOutputDeny(t *testing.T) { +func TestVector96TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_output_deny", "description": "§13.4 Guardrails — Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.", @@ -4825,7 +4854,7 @@ func TestVector95TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector96TurnConformanceRunGuardrailToolDeny(t *testing.T) { +func TestVector97TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_tool_deny", "description": "§13.4 Guardrails — LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.", @@ -4980,7 +5009,7 @@ func TestVector96TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector97TurnConformanceRunMaxIterationsExceeded(t *testing.T) { +func TestVector98TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorJSON := `{ "name": "max_iterations_exceeded", "description": "LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.", @@ -5491,7 +5520,7 @@ func TestVector97TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector98TurnConformanceRunMultiTurnToolCalls(t *testing.T) { +func TestVector99TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorJSON := `{ "name": "multi_turn_tool_calls", "description": "Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.", @@ -5669,7 +5698,7 @@ func TestVector98TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector99TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { +func TestVector100TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorJSON := `{ "name": "multiple_tool_calls_single_turn", "description": "Turn 1: LLM returns two parallel tool calls — get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.", @@ -5851,7 +5880,7 @@ func TestVector99TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector100TurnConformanceRunNoToolCalls(t *testing.T) { +func TestVector101TurnConformanceRunNoToolCalls(t *testing.T) { vectorJSON := `{ "name": "no_tool_calls", "description": "LLM returns content immediately with no tool calls — the agent loop completes in a single iteration.", @@ -5920,7 +5949,7 @@ func TestVector100TurnConformanceRunNoToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector101TurnConformanceRunParallelToolsBasic(t *testing.T) { +func TestVector102TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_basic", "description": "§13.6 Parallel Tools — LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.", @@ -6094,7 +6123,7 @@ func TestVector101TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector102TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { +func TestVector103TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_guardrail_deny", "description": "§13.6 Parallel Tools + §13.4 Guardrails — 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.", @@ -6286,7 +6315,7 @@ func TestVector102TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector103TurnConformanceRunParallelToolsWithToolError(t *testing.T) { +func TestVector104TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_tool_error", "description": "§13.6 Parallel Tools + tool error — 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.", @@ -6460,7 +6489,7 @@ func TestVector103TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector104TurnConformanceRunSingleToolCall(t *testing.T) { +func TestVector105TurnConformanceRunSingleToolCall(t *testing.T) { vectorJSON := `{ "name": "single_tool_call", "description": "Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.", @@ -6608,7 +6637,7 @@ func TestVector104TurnConformanceRunSingleToolCall(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector105TurnConformanceRunSteeringInjectMessage(t *testing.T) { +func TestVector106TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorJSON := `{ "name": "steering_inject_message", "description": "§13.5 Steering — After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.", @@ -6817,7 +6846,7 @@ func TestVector105TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector106TurnConformanceRunSteeringMultipleMessages(t *testing.T) { +func TestVector107TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorJSON := `{ "name": "steering_multiple_messages", "description": "§13.5 Steering — Two steering messages are queued and both injected before iteration 2. Tests atomic drain — both appear in order.", @@ -6988,7 +7017,7 @@ func TestVector106TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector107TurnConformanceRunToolNotRegisteredError(t *testing.T) { +func TestVector108TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorJSON := `{ "name": "tool_not_registered_error", "description": "LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.", @@ -7073,7 +7102,7 @@ func TestVector107TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector108TurnConformanceRunToolResultMessageFormat(t *testing.T) { +func TestVector109TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorJSON := `{ "name": "tool_result_message_format", "description": "After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.", @@ -7195,7 +7224,7 @@ func TestVector108TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector109TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { +func TestVector110TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorJSON := `{ "name": "cancel_before_context", "stage": "turn", @@ -7228,7 +7257,7 @@ func TestVector109TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector110TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { +func TestVector111TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorJSON := `{ "name": "delegated_provider_state", "stage": "turn", @@ -7296,7 +7325,7 @@ func TestVector110TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector111TurnConformanceRunTurnFinalOutput(t *testing.T) { +func TestVector112TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorJSON := `{ "name": "final_output", "stage": "turn", @@ -7342,7 +7371,7 @@ func TestVector111TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector112TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { +func TestVector113TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "memory_context_is_model_visible", "stage": "turn", @@ -7405,7 +7434,7 @@ func TestVector112TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector113TurnConformanceRunTurnOrderedToolRound(t *testing.T) { +func TestVector114TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorJSON := `{ "name": "ordered_tool_round", "stage": "turn", @@ -7497,7 +7526,7 @@ func TestVector113TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector114TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { +func TestVector115TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "permission_denial_is_model_visible", "stage": "turn", @@ -7549,7 +7578,7 @@ func TestVector114TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testin vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector115WireConformanceToRequestAnthropicImageFormat(t *testing.T) { +func TestVector116WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorJSON := `{ "name": "anthropic_image_format", "description": "§7.5 — Anthropic: ImagePart uses base64 source block with media_type, not image_url format.", @@ -7620,7 +7649,7 @@ func TestVector115WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector116WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { +func TestVector117WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { vectorJSON := `{ "name": "anthropic_max_tokens_required", "description": "§7.5 — Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).", @@ -7676,7 +7705,7 @@ func TestVector116WireConformanceToRequestAnthropicMaxTokensRequired(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector117WireConformanceToRequestAnthropicOptions(t *testing.T) { +func TestVector118WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorJSON := `{ "name": "anthropic_options", "description": "Anthropic: option names map correctly (topK, stopSequences)", @@ -7746,7 +7775,7 @@ func TestVector117WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector118WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { +func TestVector119WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { vectorJSON := `{ "name": "anthropic_system_separate", "description": "§7.5 — Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.", @@ -7814,7 +7843,7 @@ func TestVector118WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector119WireConformanceToRequestAnthropicToolWire(t *testing.T) { +func TestVector120WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorJSON := `{ "name": "anthropic_tool_wire", "description": "Anthropic: tools use input_schema (not parameters), name at top level", @@ -7900,7 +7929,7 @@ func TestVector119WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector120WireConformanceToRequestChatAudioMp3(t *testing.T) { +func TestVector121WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorJSON := `{ "name": "chat_audio_mp3", "description": "§7.1.2 — AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.", @@ -7959,7 +7988,7 @@ func TestVector120WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector121WireConformanceToRequestChatAudioPart(t *testing.T) { +func TestVector122WireConformanceToRequestChatAudioPart(t *testing.T) { vectorJSON := `{ "name": "chat_audio_part", "description": "§7.1.2 — AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.", @@ -8018,7 +8047,7 @@ func TestVector121WireConformanceToRequestChatAudioPart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector122WireConformanceToRequestChatImageBase64(t *testing.T) { +func TestVector123WireConformanceToRequestChatImageBase64(t *testing.T) { vectorJSON := `{ "name": "chat_image_base64", "description": "Base64 data URI image part preserved as-is in image_url", @@ -8083,7 +8112,7 @@ func TestVector122WireConformanceToRequestChatImageBase64(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector123WireConformanceToRequestChatImagePart(t *testing.T) { +func TestVector124WireConformanceToRequestChatImagePart(t *testing.T) { vectorJSON := `{ "name": "chat_image_part", "description": "Image content part converted to OpenAI image_url format", @@ -8148,7 +8177,7 @@ func TestVector123WireConformanceToRequestChatImagePart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector124WireConformanceToRequestChatMultipartContent(t *testing.T) { +func TestVector125WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorJSON := `{ "name": "chat_multipart_content", "description": "§7.1.1/§7.1.2 — Message with TextPart + ImagePart MUST produce an array of typed content blocks.", @@ -8213,7 +8242,7 @@ func TestVector124WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector125WireConformanceToRequestChatSimple(t *testing.T) { +func TestVector126WireConformanceToRequestChatSimple(t *testing.T) { vectorJSON := `{ "name": "chat_simple", "description": "§7.1.7 — Two messages (system + user), no tools, no options. Minimal chat request.", @@ -8276,7 +8305,7 @@ func TestVector125WireConformanceToRequestChatSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector126WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { +func TestVector127WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { vectorJSON := `{ "name": "chat_single_text_optimized", "description": "§7.1.1 — Message with exactly 1 TextPart MUST produce a plain string content, not an array.", @@ -8326,7 +8355,7 @@ func TestVector126WireConformanceToRequestChatSingleTextOptimized(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector127WireConformanceToRequestChatWithOptions(t *testing.T) { +func TestVector128WireConformanceToRequestChatWithOptions(t *testing.T) { vectorJSON := `{ "name": "chat_with_options", "description": "§7.1.5 — All standard ModelOptions mapped to OpenAI parameter names.", @@ -8395,7 +8424,7 @@ func TestVector127WireConformanceToRequestChatWithOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector128WireConformanceToRequestEmbeddingWire(t *testing.T) { +func TestVector129WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorJSON := `{ "name": "embedding_wire", "description": "§7.2 — Embedding request: text extracted from messages, single string input (not array).", @@ -8440,7 +8469,7 @@ func TestVector128WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector129WireConformanceToRequestImageWire(t *testing.T) { +func TestVector130WireConformanceToRequestImageWire(t *testing.T) { vectorJSON := `{ "name": "image_wire", "description": "§7.3 — Image generation request: prompt extracted from last user message.", @@ -8485,7 +8514,7 @@ func TestVector129WireConformanceToRequestImageWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector130WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { +func TestVector131WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorJSON := `{ "name": "kind_to_json_type_mapping", "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.", @@ -8605,7 +8634,7 @@ func TestVector130WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector131WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { +func TestVector132WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { vectorJSON := `{ "name": "options_additional_properties", "description": "§7.1.5 — additionalProperties from ModelOptions MUST be merged into the request as top-level keys.", @@ -8660,7 +8689,7 @@ func TestVector131WireConformanceToRequestOptionsAdditionalProperties(t *testing vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector132WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { +func TestVector133WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { vectorJSON := `{ "name": "options_max_completion_tokens", "description": "§7.1.5 — maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.", @@ -8713,7 +8742,7 @@ func TestVector132WireConformanceToRequestOptionsMaxCompletionTokens(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector133WireConformanceToRequestOptionsStopSequences(t *testing.T) { +func TestVector134WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorJSON := `{ "name": "options_stop_sequences", "description": "§7.1.5 — stopSequences MUST map to 'stop' in the OpenAI wire format.", @@ -8772,7 +8801,7 @@ func TestVector133WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector134WireConformanceToRequestResponsesSimple(t *testing.T) { +func TestVector135WireConformanceToRequestResponsesSimple(t *testing.T) { vectorJSON := `{ "name": "responses_simple", "description": "Responses API: system becomes instructions, messages become input items", @@ -8832,7 +8861,7 @@ func TestVector134WireConformanceToRequestResponsesSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector135WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { +func TestVector136WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "responses_structured_output", "description": "Responses API: structured output uses text.format.json_schema", @@ -8920,7 +8949,7 @@ func TestVector135WireConformanceToRequestResponsesStructuredOutput(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector136WireConformanceToRequestResponsesWithTools(t *testing.T) { +func TestVector137WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorJSON := `{ "name": "responses_with_tools", "description": "Responses API: tools use flat format (no nested function key)", @@ -9001,7 +9030,7 @@ func TestVector136WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector137WireConformanceToRequestStructuredOutput(t *testing.T) { +func TestVector138WireConformanceToRequestStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "structured_output", "description": "§7.1.6 — outputs converted to response_format with json_schema, strict=true, additionalProperties=false.", @@ -9085,7 +9114,7 @@ func TestVector137WireConformanceToRequestStructuredOutput(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector138WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { +func TestVector139WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { vectorJSON := `{ "name": "structured_output_nested_optional", "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", @@ -9188,7 +9217,7 @@ func TestVector138WireConformanceToRequestStructuredOutputNestedOptional(t *test vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector139WireConformanceToRequestToolsBindingsStripped(t *testing.T) { +func TestVector140WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorJSON := `{ "name": "tools_bindings_stripped", "description": "§7.1.3 — Parameters listed in bindings MUST be stripped from wire tools (properties AND required).", @@ -9281,7 +9310,7 @@ func TestVector139WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector140WireConformanceToRequestToolsFunctionWire(t *testing.T) { +func TestVector141WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorJSON := `{ "name": "tools_function_wire", "description": "§7.1.3 — FunctionTool projected as OpenAI function definition with JSON Schema parameters.", @@ -9364,7 +9393,7 @@ func TestVector140WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector141WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { +func TestVector142WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorJSON := `{ "name": "tools_null_when_empty", "description": "§7.1.3 — When no tools are defined, the tools key MUST be absent from the request entirely.", @@ -9414,7 +9443,7 @@ func TestVector141WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector142WireConformanceToRequestToolsStrictMode(t *testing.T) { +func TestVector143WireConformanceToRequestToolsStrictMode(t *testing.T) { vectorJSON := `{ "name": "tools_strict_mode", "description": "§7.1.3 — FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.", diff --git a/runtime/go/prompty/model/vectorconformance/vector_conformance.go b/runtime/go/prompty/model/vectorconformance/vector_conformance.go index 1ea2dca28..0e8fcea69 100644 --- a/runtime/go/prompty/model/vectorconformance/vector_conformance.go +++ b/runtime/go/prompty/model/vectorconformance/vector_conformance.go @@ -1254,6 +1254,17 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("untrusted_endpoint_ambient_rejected: expected an error, got nil") } } + // vector: untrusted_endpoint_backslash_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_backslash_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_backslash_rejected: expected an error, got nil") + } + } // vector: untrusted_endpoint_env_default_rejected { var source interface{} diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java index d479f6325..99580a6f3 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java @@ -50,6 +50,10 @@ record Origin(String scheme, String host, int port) {} * another. The rules follow the reference runtime's {@code urlsplit}-based parsing. */ static Origin origin(String url) { + // HTTP clients treat '\' as '/', so parsers disagree on the host; reject outright. + if (url.indexOf('\\') >= 0) { + return null; + } String s = url.strip().replace("\t", "").replace("\r", "").replace("\n", ""); int colon = s.indexOf(':'); if (colon <= 0 || !isScheme(s.substring(0, colon))) { diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java index ddd7ae655..06edf857f 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java @@ -692,6 +692,20 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex } if (caught == null) throw new AssertionError("untrusted_endpoint_ambient_rejected: expected an error"); } + { + // vector: untrusted_endpoint_backslash_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_backslash_rejected: expected an error"); + } { // vector: untrusted_endpoint_env_default_rejected StringBuilder inputJson = new StringBuilder(); diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java index bbd28f68a..ac2d8dcc0 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java @@ -442,616 +442,623 @@ private static void vector54LoadConformanceLoadUntrustedEndpointAmbientRejected( VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() throws Exception { + private static void vector55LoadConformanceLoadUntrustedEndpointBackslashRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_backslash_rejected\",\"description\":\"A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected() throws Exception { + private static void vector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected() throws Exception { + private static void vector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() throws Exception { + private static void vector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() throws Exception { + private static void vector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector60LoadConformanceLoadUntrustedEndpointNoKeyRejected() throws Exception { + private static void vector61LoadConformanceLoadUntrustedEndpointNoKeyRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() throws Exception { + private static void vector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected() throws Exception { + private static void vector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector63MemoryConformanceOperateClearByCategoryAndAll() throws Exception { + private static void vector64MemoryConformanceOperateClearByCategoryAndAll() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector64MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { + private static void vector65MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector65MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { + private static void vector66MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector66MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { + private static void vector67MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { + private static void vector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector68MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { + private static void vector69MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { + private static void vector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\\u00e9\"]},{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\\u00e9\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector70MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { + private static void vector71MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { + private static void vector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector72MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { + private static void vector73MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector73MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { + private static void vector74MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector74MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { + private static void vector75MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector75TurnConformanceReplayMaxIterations() throws Exception { + private static void vector76TurnConformanceReplayMaxIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector76TurnConformanceReplayNoTool() throws Exception { + private static void vector77TurnConformanceReplayNoTool() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector77TurnConformanceReplayPermissionDenied() throws Exception { + private static void vector78TurnConformanceReplayPermissionDenied() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector78TurnConformanceReplayToolFailure() throws Exception { + private static void vector79TurnConformanceReplayToolFailure() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector79TurnConformanceReplayToolSuccess() throws Exception { + private static void vector80TurnConformanceReplayToolSuccess() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector80TurnConformanceRunAssistantToolCallsMetadata() throws Exception { + private static void vector81TurnConformanceRunAssistantToolCallsMetadata() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector81TurnConformanceRunAsyncToolFunction() throws Exception { + private static void vector82TurnConformanceRunAsyncToolFunction() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector82TurnConformanceRunBindingsInjected() throws Exception { + private static void vector83TurnConformanceRunBindingsInjected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\\u00b0C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector83TurnConformanceRunCancellationBeforeLlm() throws Exception { + private static void vector84TurnConformanceRunCancellationBeforeLlm() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_before_llm\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector84TurnConformanceRunCancellationBetweenIterations() throws Exception { + private static void vector85TurnConformanceRunCancellationBetweenIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_iterations\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector85TurnConformanceRunCancellationBetweenTools() throws Exception { + private static void vector86TurnConformanceRunCancellationBetweenTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_tools\",\"description\":\"\\u00a713.2 Cancellation \\u2014 LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector86TurnConformanceRunContextNoTrimWhenFits() throws Exception { + private static void vector87TurnConformanceRunContextNoTrimWhenFits() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector87TurnConformanceRunContextPreservesSystemMessages() throws Exception { + private static void vector88TurnConformanceRunContextPreservesSystemMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_preserves_system_messages\",\"description\":\"\\u00a713.3 Context Window \\u2014 Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \\u2014 they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector88TurnConformanceRunContextTrimBasic() throws Exception { + private static void vector89TurnConformanceRunContextTrimBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_trim_basic\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector89TurnConformanceRunEmptyToolResult() throws Exception { + private static void vector90TurnConformanceRunEmptyToolResult() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \\u2014 it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector90TurnConformanceRunEventsBasicToolLoop() throws Exception { + private static void vector91TurnConformanceRunEventsBasicToolLoop() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_basic_tool_loop\",\"description\":\"\\u00a713.1 Events \\u2014 A 2-turn tool-call loop emits the full event lifecycle: status \\u2192 tool_call_start \\u2192 tool_result \\u2192 messages_updated \\u2192 done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\\u00b0F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector91TurnConformanceRunEventsErrorLogged() throws Exception { + private static void vector92TurnConformanceRunEventsErrorLogged() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_error_logged\",\"description\":\"\\u00a713.1 Events \\u0097 A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \\u0097 this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector92TurnConformanceRunEventsNoTools() throws Exception { + private static void vector93TurnConformanceRunEventsNoTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_no_tools\",\"description\":\"\\u00a713.1 Events \\u2014 Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector93TurnConformanceRunGuardrailAllPass() throws Exception { + private static void vector94TurnConformanceRunGuardrailAllPass() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_all_pass\",\"description\":\"\\u00a713.4 Guardrails \\u2014 All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector94TurnConformanceRunGuardrailInputDeny() throws Exception { + private static void vector95TurnConformanceRunGuardrailInputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_input_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector95TurnConformanceRunGuardrailOutputDeny() throws Exception { + private static void vector96TurnConformanceRunGuardrailOutputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_output_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector96TurnConformanceRunGuardrailToolDeny() throws Exception { + private static void vector97TurnConformanceRunGuardrailToolDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_tool_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \\u2014 its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector97TurnConformanceRunMaxIterationsExceeded() throws Exception { + private static void vector98TurnConformanceRunMaxIterationsExceeded() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\\u00b0F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\\u00b0F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\\u00b0F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\\u00b0F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\\u00b0F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\\u00b0F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\\u00b0F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\\u00b0F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\\u00b0F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector98TurnConformanceRunMultiTurnToolCalls() throws Exception { + private static void vector99TurnConformanceRunMultiTurnToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\\u00b0C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector99TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { + private static void vector100TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \\u2014 get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\\u00b0F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector100TurnConformanceRunNoToolCalls() throws Exception { + private static void vector101TurnConformanceRunNoToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \\u2014 the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector101TurnConformanceRunParallelToolsBasic() throws Exception { + private static void vector102TurnConformanceRunParallelToolsBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_basic\",\"description\":\"\\u00a713.6 Parallel Tools \\u2014 LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector102TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { + private static void vector103TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\\u00a713.6 Parallel Tools + \\u00a713.4 Guardrails \\u2014 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector103TurnConformanceRunParallelToolsWithToolError() throws Exception { + private static void vector104TurnConformanceRunParallelToolsWithToolError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\\u00a713.6 Parallel Tools + tool error \\u2014 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector104TurnConformanceRunSingleToolCall() throws Exception { + private static void vector105TurnConformanceRunSingleToolCall() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector105TurnConformanceRunSteeringInjectMessage() throws Exception { + private static void vector106TurnConformanceRunSteeringInjectMessage() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_inject_message\",\"description\":\"\\u00a713.5 Steering \\u2014 After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector106TurnConformanceRunSteeringMultipleMessages() throws Exception { + private static void vector107TurnConformanceRunSteeringMultipleMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_multiple_messages\",\"description\":\"\\u00a713.5 Steering \\u2014 Two steering messages are queued and both injected before iteration 2. Tests atomic drain \\u2014 both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\\u00b0F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector107TurnConformanceRunToolNotRegisteredError() throws Exception { + private static void vector108TurnConformanceRunToolNotRegisteredError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector108TurnConformanceRunToolResultMessageFormat() throws Exception { + private static void vector109TurnConformanceRunToolResultMessageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\\u00b0F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\\u00b0F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector109TurnConformanceRunTurnCancelBeforeContext() throws Exception { + private static void vector110TurnConformanceRunTurnCancelBeforeContext() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector110TurnConformanceRunTurnDelegatedProviderState() throws Exception { + private static void vector111TurnConformanceRunTurnDelegatedProviderState() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector111TurnConformanceRunTurnFinalOutput() throws Exception { + private static void vector112TurnConformanceRunTurnFinalOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector112TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { + private static void vector113TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector113TurnConformanceRunTurnOrderedToolRound() throws Exception { + private static void vector114TurnConformanceRunTurnOrderedToolRound() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector114TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { + private static void vector115TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector115WireConformanceToRequestAnthropicImageFormat() throws Exception { + private static void vector116WireConformanceToRequestAnthropicImageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_image_format\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector116WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { + private static void vector117WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector117WireConformanceToRequestAnthropicOptions() throws Exception { + private static void vector118WireConformanceToRequestAnthropicOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector118WireConformanceToRequestAnthropicSystemSeparate() throws Exception { + private static void vector119WireConformanceToRequestAnthropicSystemSeparate() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_system_separate\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector119WireConformanceToRequestAnthropicToolWire() throws Exception { + private static void vector120WireConformanceToRequestAnthropicToolWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector120WireConformanceToRequestChatAudioMp3() throws Exception { + private static void vector121WireConformanceToRequestChatAudioMp3() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_mp3\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector121WireConformanceToRequestChatAudioPart() throws Exception { + private static void vector122WireConformanceToRequestChatAudioPart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_part\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector122WireConformanceToRequestChatImageBase64() throws Exception { + private static void vector123WireConformanceToRequestChatImageBase64() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector123WireConformanceToRequestChatImagePart() throws Exception { + private static void vector124WireConformanceToRequestChatImagePart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector124WireConformanceToRequestChatMultipartContent() throws Exception { + private static void vector125WireConformanceToRequestChatMultipartContent() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_multipart_content\",\"description\":\"\\u00a77.1.1/\\u00a77.1.2 \\u2014 Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector125WireConformanceToRequestChatSimple() throws Exception { + private static void vector126WireConformanceToRequestChatSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_simple\",\"description\":\"\\u00a77.1.7 \\u2014 Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector126WireConformanceToRequestChatSingleTextOptimized() throws Exception { + private static void vector127WireConformanceToRequestChatSingleTextOptimized() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_single_text_optimized\",\"description\":\"\\u00a77.1.1 \\u2014 Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector127WireConformanceToRequestChatWithOptions() throws Exception { + private static void vector128WireConformanceToRequestChatWithOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_with_options\",\"description\":\"\\u00a77.1.5 \\u2014 All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector128WireConformanceToRequestEmbeddingWire() throws Exception { + private static void vector129WireConformanceToRequestEmbeddingWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"embedding_wire\",\"description\":\"\\u00a77.2 \\u2014 Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector129WireConformanceToRequestImageWire() throws Exception { + private static void vector130WireConformanceToRequestImageWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"image_wire\",\"description\":\"\\u00a77.3 \\u2014 Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector130WireConformanceToRequestKindToJsonTypeMapping() throws Exception { + private static void vector131WireConformanceToRequestKindToJsonTypeMapping() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\\u00a77.1.4 \\u2014 All Property kind values mapped to JSON Schema types: string\\u2192string, integer\\u2192integer, float\\u2192number, boolean\\u2192boolean, array\\u2192array, object\\u2192object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector131WireConformanceToRequestOptionsAdditionalProperties() throws Exception { + private static void vector132WireConformanceToRequestOptionsAdditionalProperties() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_additional_properties\",\"description\":\"\\u00a77.1.5 \\u2014 additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector132WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { + private static void vector133WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_max_completion_tokens\",\"description\":\"\\u00a77.1.5 \\u2014 maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector133WireConformanceToRequestOptionsStopSequences() throws Exception { + private static void vector134WireConformanceToRequestOptionsStopSequences() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_stop_sequences\",\"description\":\"\\u00a77.1.5 \\u2014 stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector134WireConformanceToRequestResponsesSimple() throws Exception { + private static void vector135WireConformanceToRequestResponsesSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector135WireConformanceToRequestResponsesStructuredOutput() throws Exception { + private static void vector136WireConformanceToRequestResponsesStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector136WireConformanceToRequestResponsesWithTools() throws Exception { + private static void vector137WireConformanceToRequestResponsesWithTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector137WireConformanceToRequestStructuredOutput() throws Exception { + private static void vector138WireConformanceToRequestStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output\",\"description\":\"\\u00a77.1.6 \\u2014 outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector138WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { + private static void vector139WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output_nested_optional\",\"description\":\"\\u00a77.1.4/\\u00a77.1.6 \\u2014 OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector139WireConformanceToRequestToolsBindingsStripped() throws Exception { + private static void vector140WireConformanceToRequestToolsBindingsStripped() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_bindings_stripped\",\"description\":\"\\u00a77.1.3 \\u2014 Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector140WireConformanceToRequestToolsFunctionWire() throws Exception { + private static void vector141WireConformanceToRequestToolsFunctionWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_function_wire\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector141WireConformanceToRequestToolsNullWhenEmpty() throws Exception { + private static void vector142WireConformanceToRequestToolsNullWhenEmpty() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_null_when_empty\",\"description\":\"\\u00a77.1.3 \\u2014 When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector142WireConformanceToRequestToolsStrictMode() throws Exception { + private static void vector143WireConformanceToRequestToolsStrictMode() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_strict_mode\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); @@ -1115,94 +1122,95 @@ public static void run() { failed += runCase("LoadConformance.load:trusted_endpoint_option_allowed", VectorConformanceTests::vector52LoadConformanceLoadTrustedEndpointOptionAllowed); failed += runCase("LoadConformance.load:trusted_endpoint_reference_skipped", VectorConformanceTests::vector53LoadConformanceLoadTrustedEndpointReferenceSkipped); failed += runCase("LoadConformance.load:untrusted_endpoint_ambient_rejected", VectorConformanceTests::vector54LoadConformanceLoadUntrustedEndpointAmbientRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_env_default_rejected", VectorConformanceTests::vector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_env_key_rejected", VectorConformanceTests::vector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_file_connection_rejected", VectorConformanceTests::vector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", VectorConformanceTests::vector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", VectorConformanceTests::vector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_no_key_rejected", VectorConformanceTests::vector60LoadConformanceLoadUntrustedEndpointNoKeyRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", VectorConformanceTests::vector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", VectorConformanceTests::vector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected); - failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector63MemoryConformanceOperateClearByCategoryAndAll); - failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector64MemoryConformanceOperateFormatCoreAndRecallResults); - failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector65MemoryConformanceOperateRecallEmptyQueryAndLimit); - failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector66MemoryConformanceOperateRecallRanksByWeightedScore); - failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); - failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector68MemoryConformanceOperateRecallTagsAndCoreBoost); - failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); - failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector70MemoryConformanceOperateRememberDedupsCoreByTags); - failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); - failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector72MemoryConformanceOperateRememberEvictsArchivalFirst); - failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector73MemoryConformanceOperateSnapshotPortRoundTrip); - failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector74MemoryConformanceOperateUpdateIndexOutOfRangeError); - failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector75TurnConformanceReplayMaxIterations); - failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector76TurnConformanceReplayNoTool); - failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector77TurnConformanceReplayPermissionDenied); - failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector78TurnConformanceReplayToolFailure); - failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector79TurnConformanceReplayToolSuccess); - failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector80TurnConformanceRunAssistantToolCallsMetadata); - failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector81TurnConformanceRunAsyncToolFunction); - failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector82TurnConformanceRunBindingsInjected); - failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector83TurnConformanceRunCancellationBeforeLlm); - failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector84TurnConformanceRunCancellationBetweenIterations); - failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector85TurnConformanceRunCancellationBetweenTools); - failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector86TurnConformanceRunContextNoTrimWhenFits); - failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector87TurnConformanceRunContextPreservesSystemMessages); - failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector88TurnConformanceRunContextTrimBasic); - failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector89TurnConformanceRunEmptyToolResult); - failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector90TurnConformanceRunEventsBasicToolLoop); - failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector91TurnConformanceRunEventsErrorLogged); - failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector92TurnConformanceRunEventsNoTools); - failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector93TurnConformanceRunGuardrailAllPass); - failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector94TurnConformanceRunGuardrailInputDeny); - failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector95TurnConformanceRunGuardrailOutputDeny); - failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector96TurnConformanceRunGuardrailToolDeny); - failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector97TurnConformanceRunMaxIterationsExceeded); - failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector98TurnConformanceRunMultiTurnToolCalls); - failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector99TurnConformanceRunMultipleToolCallsSingleTurn); - failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector100TurnConformanceRunNoToolCalls); - failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector101TurnConformanceRunParallelToolsBasic); - failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector102TurnConformanceRunParallelToolsWithGuardrailDeny); - failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector103TurnConformanceRunParallelToolsWithToolError); - failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector104TurnConformanceRunSingleToolCall); - failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector105TurnConformanceRunSteeringInjectMessage); - failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector106TurnConformanceRunSteeringMultipleMessages); - failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector107TurnConformanceRunToolNotRegisteredError); - failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector108TurnConformanceRunToolResultMessageFormat); - failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector109TurnConformanceRunTurnCancelBeforeContext); - failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector110TurnConformanceRunTurnDelegatedProviderState); - failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector111TurnConformanceRunTurnFinalOutput); - failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector112TurnConformanceRunTurnMemoryContextIsModelVisible); - failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector113TurnConformanceRunTurnOrderedToolRound); - failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector114TurnConformanceRunTurnPermissionDenialIsModelVisible); - failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector115WireConformanceToRequestAnthropicImageFormat); - failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector116WireConformanceToRequestAnthropicMaxTokensRequired); - failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector117WireConformanceToRequestAnthropicOptions); - failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector118WireConformanceToRequestAnthropicSystemSeparate); - failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector119WireConformanceToRequestAnthropicToolWire); - failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector120WireConformanceToRequestChatAudioMp3); - failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector121WireConformanceToRequestChatAudioPart); - failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector122WireConformanceToRequestChatImageBase64); - failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector123WireConformanceToRequestChatImagePart); - failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector124WireConformanceToRequestChatMultipartContent); - failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector125WireConformanceToRequestChatSimple); - failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector126WireConformanceToRequestChatSingleTextOptimized); - failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector127WireConformanceToRequestChatWithOptions); - failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector128WireConformanceToRequestEmbeddingWire); - failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector129WireConformanceToRequestImageWire); - failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector130WireConformanceToRequestKindToJsonTypeMapping); - failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector131WireConformanceToRequestOptionsAdditionalProperties); - failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector132WireConformanceToRequestOptionsMaxCompletionTokens); - failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector133WireConformanceToRequestOptionsStopSequences); - failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector134WireConformanceToRequestResponsesSimple); - failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector135WireConformanceToRequestResponsesStructuredOutput); - failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector136WireConformanceToRequestResponsesWithTools); - failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector137WireConformanceToRequestStructuredOutput); - failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector138WireConformanceToRequestStructuredOutputNestedOptional); - failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector139WireConformanceToRequestToolsBindingsStripped); - failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector140WireConformanceToRequestToolsFunctionWire); - failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector141WireConformanceToRequestToolsNullWhenEmpty); - failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector142WireConformanceToRequestToolsStrictMode); + failed += runCase("LoadConformance.load:untrusted_endpoint_backslash_rejected", VectorConformanceTests::vector55LoadConformanceLoadUntrustedEndpointBackslashRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_env_default_rejected", VectorConformanceTests::vector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_env_key_rejected", VectorConformanceTests::vector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_file_connection_rejected", VectorConformanceTests::vector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", VectorConformanceTests::vector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", VectorConformanceTests::vector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_no_key_rejected", VectorConformanceTests::vector61LoadConformanceLoadUntrustedEndpointNoKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", VectorConformanceTests::vector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", VectorConformanceTests::vector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected); + failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector64MemoryConformanceOperateClearByCategoryAndAll); + failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector65MemoryConformanceOperateFormatCoreAndRecallResults); + failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector66MemoryConformanceOperateRecallEmptyQueryAndLimit); + failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector67MemoryConformanceOperateRecallRanksByWeightedScore); + failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); + failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector69MemoryConformanceOperateRecallTagsAndCoreBoost); + failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); + failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector71MemoryConformanceOperateRememberDedupsCoreByTags); + failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); + failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector73MemoryConformanceOperateRememberEvictsArchivalFirst); + failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector74MemoryConformanceOperateSnapshotPortRoundTrip); + failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector75MemoryConformanceOperateUpdateIndexOutOfRangeError); + failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector76TurnConformanceReplayMaxIterations); + failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector77TurnConformanceReplayNoTool); + failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector78TurnConformanceReplayPermissionDenied); + failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector79TurnConformanceReplayToolFailure); + failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector80TurnConformanceReplayToolSuccess); + failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector81TurnConformanceRunAssistantToolCallsMetadata); + failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector82TurnConformanceRunAsyncToolFunction); + failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector83TurnConformanceRunBindingsInjected); + failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector84TurnConformanceRunCancellationBeforeLlm); + failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector85TurnConformanceRunCancellationBetweenIterations); + failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector86TurnConformanceRunCancellationBetweenTools); + failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector87TurnConformanceRunContextNoTrimWhenFits); + failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector88TurnConformanceRunContextPreservesSystemMessages); + failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector89TurnConformanceRunContextTrimBasic); + failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector90TurnConformanceRunEmptyToolResult); + failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector91TurnConformanceRunEventsBasicToolLoop); + failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector92TurnConformanceRunEventsErrorLogged); + failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector93TurnConformanceRunEventsNoTools); + failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector94TurnConformanceRunGuardrailAllPass); + failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector95TurnConformanceRunGuardrailInputDeny); + failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector96TurnConformanceRunGuardrailOutputDeny); + failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector97TurnConformanceRunGuardrailToolDeny); + failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector98TurnConformanceRunMaxIterationsExceeded); + failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector99TurnConformanceRunMultiTurnToolCalls); + failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector100TurnConformanceRunMultipleToolCallsSingleTurn); + failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector101TurnConformanceRunNoToolCalls); + failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector102TurnConformanceRunParallelToolsBasic); + failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector103TurnConformanceRunParallelToolsWithGuardrailDeny); + failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector104TurnConformanceRunParallelToolsWithToolError); + failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector105TurnConformanceRunSingleToolCall); + failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector106TurnConformanceRunSteeringInjectMessage); + failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector107TurnConformanceRunSteeringMultipleMessages); + failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector108TurnConformanceRunToolNotRegisteredError); + failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector109TurnConformanceRunToolResultMessageFormat); + failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector110TurnConformanceRunTurnCancelBeforeContext); + failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector111TurnConformanceRunTurnDelegatedProviderState); + failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector112TurnConformanceRunTurnFinalOutput); + failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector113TurnConformanceRunTurnMemoryContextIsModelVisible); + failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector114TurnConformanceRunTurnOrderedToolRound); + failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector115TurnConformanceRunTurnPermissionDenialIsModelVisible); + failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector116WireConformanceToRequestAnthropicImageFormat); + failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector117WireConformanceToRequestAnthropicMaxTokensRequired); + failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector118WireConformanceToRequestAnthropicOptions); + failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector119WireConformanceToRequestAnthropicSystemSeparate); + failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector120WireConformanceToRequestAnthropicToolWire); + failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector121WireConformanceToRequestChatAudioMp3); + failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector122WireConformanceToRequestChatAudioPart); + failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector123WireConformanceToRequestChatImageBase64); + failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector124WireConformanceToRequestChatImagePart); + failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector125WireConformanceToRequestChatMultipartContent); + failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector126WireConformanceToRequestChatSimple); + failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector127WireConformanceToRequestChatSingleTextOptimized); + failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector128WireConformanceToRequestChatWithOptions); + failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector129WireConformanceToRequestEmbeddingWire); + failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector130WireConformanceToRequestImageWire); + failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector131WireConformanceToRequestKindToJsonTypeMapping); + failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector132WireConformanceToRequestOptionsAdditionalProperties); + failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector133WireConformanceToRequestOptionsMaxCompletionTokens); + failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector134WireConformanceToRequestOptionsStopSequences); + failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector135WireConformanceToRequestResponsesSimple); + failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector136WireConformanceToRequestResponsesStructuredOutput); + failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector137WireConformanceToRequestResponsesWithTools); + failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector138WireConformanceToRequestStructuredOutput); + failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector139WireConformanceToRequestStructuredOutputNestedOptional); + failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector140WireConformanceToRequestToolsBindingsStripped); + failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector141WireConformanceToRequestToolsFunctionWire); + failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector142WireConformanceToRequestToolsNullWhenEmpty); + failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector143WireConformanceToRequestToolsStrictMode); if (failed > 0) { throw new AssertionError(failed + " @vector conformance failure(s)"); } diff --git a/runtime/python/prompty/prompty/core/endpoint_trust.py b/runtime/python/prompty/prompty/core/endpoint_trust.py index f952affb5..dc2dd235a 100644 --- a/runtime/python/prompty/prompty/core/endpoint_trust.py +++ b/runtime/python/prompty/prompty/core/endpoint_trust.py @@ -35,6 +35,9 @@ def _origin(url: str) -> tuple[str, str, int] | None: """Return ``(scheme, host, port)`` for an absolute http(s) URL, else ``None``.""" + # HTTP clients treat '\' as '/', so parsers disagree on the host; reject outright. + if "\\" in url: + return None try: parts = urlsplit(url.strip()) port = parts.port diff --git a/runtime/python/prompty/prompty/model/vector_conformance.py b/runtime/python/prompty/prompty/model/vector_conformance.py index eb796a5b3..12b28c1ab 100644 --- a/runtime/python/prompty/prompty/model/vector_conformance.py +++ b/runtime/python/prompty/prompty/model/vector_conformance.py @@ -527,6 +527,16 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: except Exception as error: # noqa: BLE001 caught = error assert caught is not None, "untrusted_endpoint_ambient_rejected: expected an error" + # vector: untrusted_endpoint_backslash_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_backslash_rejected: expected an error" # vector: untrusted_endpoint_env_default_rejected source = json.loads("null") caught = None diff --git a/runtime/python/prompty/tests/model/test_vector_conformance.py b/runtime/python/prompty/tests/model/test_vector_conformance.py index 5c218561d..11709fb20 100644 --- a/runtime/python/prompty/tests/model/test_vector_conformance.py +++ b/runtime/python/prompty/tests/model/test_vector_conformance.py @@ -1938,7 +1938,35 @@ async def test_vector_54_loadconformance_load_untrusted_endpoint_ambient_rejecte await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_55_loadconformance_load_untrusted_endpoint_env_default_rejected(): +async def test_vector_55_loadconformance_load_untrusted_endpoint_backslash_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_backslash_rejected", + "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-backslash", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com\\@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_56_loadconformance_load_untrusted_endpoint_env_default_rejected(): vector_json = r""" { "name": "untrusted_endpoint_env_default_rejected", @@ -1967,7 +1995,7 @@ async def test_vector_55_loadconformance_load_untrusted_endpoint_env_default_rej await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_56_loadconformance_load_untrusted_endpoint_env_key_rejected(): +async def test_vector_57_loadconformance_load_untrusted_endpoint_env_key_rejected(): vector_json = r""" { "name": "untrusted_endpoint_env_key_rejected", @@ -1999,7 +2027,7 @@ async def test_vector_56_loadconformance_load_untrusted_endpoint_env_key_rejecte await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_57_loadconformance_load_untrusted_endpoint_file_connection_rejected(): +async def test_vector_58_loadconformance_load_untrusted_endpoint_file_connection_rejected(): vector_json = r""" { "name": "untrusted_endpoint_file_connection_rejected", @@ -2030,7 +2058,7 @@ async def test_vector_57_loadconformance_load_untrusted_endpoint_file_connection await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_58_loadconformance_load_untrusted_endpoint_key_with_client_secret_rejected(): +async def test_vector_59_loadconformance_load_untrusted_endpoint_key_with_client_secret_rejected(): vector_json = r""" { "name": "untrusted_endpoint_key_with_client_secret_rejected", @@ -2059,7 +2087,7 @@ async def test_vector_58_loadconformance_load_untrusted_endpoint_key_with_client await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_59_loadconformance_load_untrusted_endpoint_mismatched_credential_rejected(): +async def test_vector_60_loadconformance_load_untrusted_endpoint_mismatched_credential_rejected(): vector_json = r""" { "name": "untrusted_endpoint_mismatched_credential_rejected", @@ -2088,7 +2116,7 @@ async def test_vector_59_loadconformance_load_untrusted_endpoint_mismatched_cred await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_60_loadconformance_load_untrusted_endpoint_no_key_rejected(): +async def test_vector_61_loadconformance_load_untrusted_endpoint_no_key_rejected(): vector_json = r""" { "name": "untrusted_endpoint_no_key_rejected", @@ -2116,7 +2144,7 @@ async def test_vector_60_loadconformance_load_untrusted_endpoint_no_key_rejected await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_61_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected(): +async def test_vector_62_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected(): vector_json = r""" { "name": "untrusted_endpoint_suffix_spoof_rejected", @@ -2147,7 +2175,7 @@ async def test_vector_61_loadconformance_load_untrusted_endpoint_suffix_spoof_re await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_62_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected(): +async def test_vector_63_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected(): vector_json = r""" { "name": "untrusted_endpoint_wildcard_apex_rejected", @@ -2175,7 +2203,7 @@ async def test_vector_62_loadconformance_load_untrusted_endpoint_wildcard_apex_r await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_63_memoryconformance_operate_clear_by_category_and_all(): +async def test_vector_64_memoryconformance_operate_clear_by_category_and_all(): vector_json = r""" { "name": "clear_by_category_and_all", @@ -2227,7 +2255,7 @@ async def test_vector_63_memoryconformance_operate_clear_by_category_and_all(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_64_memoryconformance_operate_format_core_and_recall_results(): +async def test_vector_65_memoryconformance_operate_format_core_and_recall_results(): vector_json = r""" { "name": "format_core_and_recall_results", @@ -2267,7 +2295,7 @@ async def test_vector_64_memoryconformance_operate_format_core_and_recall_result await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_65_memoryconformance_operate_recall_empty_query_and_limit(): +async def test_vector_66_memoryconformance_operate_recall_empty_query_and_limit(): vector_json = r""" { "name": "recall_empty_query_and_limit", @@ -2320,7 +2348,7 @@ async def test_vector_65_memoryconformance_operate_recall_empty_query_and_limit( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_66_memoryconformance_operate_recall_ranks_by_weighted_score(): +async def test_vector_67_memoryconformance_operate_recall_ranks_by_weighted_score(): vector_json = r""" { "name": "recall_ranks_by_weighted_score", @@ -2377,7 +2405,7 @@ async def test_vector_66_memoryconformance_operate_recall_ranks_by_weighted_scor await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_67_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): +async def test_vector_68_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): vector_json = r""" { "name": "recall_score_ties_preserve_insertion_order", @@ -2433,7 +2461,7 @@ async def test_vector_67_memoryconformance_operate_recall_score_ties_preserve_in await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_68_memoryconformance_operate_recall_tags_and_core_boost(): +async def test_vector_69_memoryconformance_operate_recall_tags_and_core_boost(): vector_json = r""" { "name": "recall_tags_and_core_boost", @@ -2492,7 +2520,7 @@ async def test_vector_68_memoryconformance_operate_recall_tags_and_core_boost(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_69_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): +async def test_vector_70_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): vector_json = r""" { "name": "recall_tokenization_punctuation_unicode_and_dedup", @@ -2551,7 +2579,7 @@ async def test_vector_69_memoryconformance_operate_recall_tokenization_punctuati await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_70_memoryconformance_operate_remember_dedups_core_by_tags(): +async def test_vector_71_memoryconformance_operate_remember_dedups_core_by_tags(): vector_json = r""" { "name": "remember_dedups_core_by_tags", @@ -2613,7 +2641,7 @@ async def test_vector_70_memoryconformance_operate_remember_dedups_core_by_tags( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_71_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): +async def test_vector_72_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): vector_json = r""" { "name": "remember_dedups_core_missing_and_empty_tags", @@ -2667,7 +2695,7 @@ async def test_vector_71_memoryconformance_operate_remember_dedups_core_missing_ await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_72_memoryconformance_operate_remember_evicts_archival_first(): +async def test_vector_73_memoryconformance_operate_remember_evicts_archival_first(): vector_json = r""" { "name": "remember_evicts_archival_first", @@ -2726,7 +2754,7 @@ async def test_vector_72_memoryconformance_operate_remember_evicts_archival_firs await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_73_memoryconformance_operate_snapshot_port_round_trip(): +async def test_vector_74_memoryconformance_operate_snapshot_port_round_trip(): vector_json = r""" { "name": "snapshot_port_round_trip", @@ -2764,7 +2792,7 @@ async def test_vector_73_memoryconformance_operate_snapshot_port_round_trip(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_74_memoryconformance_operate_update_index_out_of_range_error(): +async def test_vector_75_memoryconformance_operate_update_index_out_of_range_error(): vector_json = r""" { "name": "update_index_out_of_range_error", @@ -2796,7 +2824,7 @@ async def test_vector_74_memoryconformance_operate_update_index_out_of_range_err await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_75_turnconformance_replay_max_iterations(): +async def test_vector_76_turnconformance_replay_max_iterations(): vector_json = r""" { "name": "max_iterations", @@ -2831,7 +2859,7 @@ async def test_vector_75_turnconformance_replay_max_iterations(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_76_turnconformance_replay_no_tool(): +async def test_vector_77_turnconformance_replay_no_tool(): vector_json = r""" { "name": "no_tool", @@ -2862,7 +2890,7 @@ async def test_vector_76_turnconformance_replay_no_tool(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_77_turnconformance_replay_permission_denied(): +async def test_vector_78_turnconformance_replay_permission_denied(): vector_json = r""" { "name": "permission_denied", @@ -2896,7 +2924,7 @@ async def test_vector_77_turnconformance_replay_permission_denied(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_78_turnconformance_replay_tool_failure(): +async def test_vector_79_turnconformance_replay_tool_failure(): vector_json = r""" { "name": "tool_failure", @@ -2932,7 +2960,7 @@ async def test_vector_78_turnconformance_replay_tool_failure(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_79_turnconformance_replay_tool_success(): +async def test_vector_80_turnconformance_replay_tool_success(): vector_json = r""" { "name": "tool_success", @@ -2968,7 +2996,7 @@ async def test_vector_79_turnconformance_replay_tool_success(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_80_turnconformance_run_assistant_tool_calls_metadata(): +async def test_vector_81_turnconformance_run_assistant_tool_calls_metadata(): vector_json = r""" { "name": "assistant_tool_calls_metadata", @@ -3120,7 +3148,7 @@ async def test_vector_80_turnconformance_run_assistant_tool_calls_metadata(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_81_turnconformance_run_async_tool_function(): +async def test_vector_82_turnconformance_run_async_tool_function(): vector_json = r""" { "name": "async_tool_function", @@ -3230,7 +3258,7 @@ async def test_vector_81_turnconformance_run_async_tool_function(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_82_turnconformance_run_bindings_injected(): +async def test_vector_83_turnconformance_run_bindings_injected(): vector_json = r""" { "name": "bindings_injected", @@ -3360,7 +3388,7 @@ async def test_vector_82_turnconformance_run_bindings_injected(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_83_turnconformance_run_cancellation_before_llm(): +async def test_vector_84_turnconformance_run_cancellation_before_llm(): vector_json = r""" { "name": "cancellation_before_llm", @@ -3421,7 +3449,7 @@ async def test_vector_83_turnconformance_run_cancellation_before_llm(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_84_turnconformance_run_cancellation_between_iterations(): +async def test_vector_85_turnconformance_run_cancellation_between_iterations(): vector_json = r""" { "name": "cancellation_between_iterations", @@ -3551,7 +3579,7 @@ async def test_vector_84_turnconformance_run_cancellation_between_iterations(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_85_turnconformance_run_cancellation_between_tools(): +async def test_vector_86_turnconformance_run_cancellation_between_tools(): vector_json = r""" { "name": "cancellation_between_tools", @@ -3690,7 +3718,7 @@ async def test_vector_85_turnconformance_run_cancellation_between_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_86_turnconformance_run_context_no_trim_when_fits(): +async def test_vector_87_turnconformance_run_context_no_trim_when_fits(): vector_json = r""" { "name": "context_no_trim_when_fits", @@ -3761,7 +3789,7 @@ async def test_vector_86_turnconformance_run_context_no_trim_when_fits(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_87_turnconformance_run_context_preserves_system_messages(): +async def test_vector_88_turnconformance_run_context_preserves_system_messages(): vector_json = r""" { "name": "context_preserves_system_messages", @@ -3868,7 +3896,7 @@ async def test_vector_87_turnconformance_run_context_preserves_system_messages() await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_88_turnconformance_run_context_trim_basic(): +async def test_vector_89_turnconformance_run_context_trim_basic(): vector_json = r""" { "name": "context_trim_basic", @@ -4026,7 +4054,7 @@ async def test_vector_88_turnconformance_run_context_trim_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_89_turnconformance_run_empty_tool_result(): +async def test_vector_90_turnconformance_run_empty_tool_result(): vector_json = r""" { "name": "empty_tool_result", @@ -4141,7 +4169,7 @@ async def test_vector_89_turnconformance_run_empty_tool_result(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_90_turnconformance_run_events_basic_tool_loop(): +async def test_vector_91_turnconformance_run_events_basic_tool_loop(): vector_json = r""" { "name": "events_basic_tool_loop", @@ -4287,7 +4315,7 @@ async def test_vector_90_turnconformance_run_events_basic_tool_loop(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_91_turnconformance_run_events_error_logged(): +async def test_vector_92_turnconformance_run_events_error_logged(): vector_json = r""" { "name": "events_error_logged", @@ -4413,7 +4441,7 @@ async def test_vector_91_turnconformance_run_events_error_logged(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_92_turnconformance_run_events_no_tools(): +async def test_vector_93_turnconformance_run_events_no_tools(): vector_json = r""" { "name": "events_no_tools", @@ -4491,7 +4519,7 @@ async def test_vector_92_turnconformance_run_events_no_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_93_turnconformance_run_guardrail_all_pass(): +async def test_vector_94_turnconformance_run_guardrail_all_pass(): vector_json = r""" { "name": "guardrail_all_pass", @@ -4614,7 +4642,7 @@ async def test_vector_93_turnconformance_run_guardrail_all_pass(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_94_turnconformance_run_guardrail_input_deny(): +async def test_vector_95_turnconformance_run_guardrail_input_deny(): vector_json = r""" { "name": "guardrail_input_deny", @@ -4670,7 +4698,7 @@ async def test_vector_94_turnconformance_run_guardrail_input_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_95_turnconformance_run_guardrail_output_deny(): +async def test_vector_96_turnconformance_run_guardrail_output_deny(): vector_json = r""" { "name": "guardrail_output_deny", @@ -4748,7 +4776,7 @@ async def test_vector_95_turnconformance_run_guardrail_output_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_96_turnconformance_run_guardrail_tool_deny(): +async def test_vector_97_turnconformance_run_guardrail_tool_deny(): vector_json = r""" { "name": "guardrail_tool_deny", @@ -4902,7 +4930,7 @@ async def test_vector_96_turnconformance_run_guardrail_tool_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_97_turnconformance_run_max_iterations_exceeded(): +async def test_vector_98_turnconformance_run_max_iterations_exceeded(): vector_json = r""" { "name": "max_iterations_exceeded", @@ -5412,7 +5440,7 @@ async def test_vector_97_turnconformance_run_max_iterations_exceeded(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_98_turnconformance_run_multi_turn_tool_calls(): +async def test_vector_99_turnconformance_run_multi_turn_tool_calls(): vector_json = r""" { "name": "multi_turn_tool_calls", @@ -5589,7 +5617,7 @@ async def test_vector_98_turnconformance_run_multi_turn_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_99_turnconformance_run_multiple_tool_calls_single_turn(): +async def test_vector_100_turnconformance_run_multiple_tool_calls_single_turn(): vector_json = r""" { "name": "multiple_tool_calls_single_turn", @@ -5770,7 +5798,7 @@ async def test_vector_99_turnconformance_run_multiple_tool_calls_single_turn(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_100_turnconformance_run_no_tool_calls(): +async def test_vector_101_turnconformance_run_no_tool_calls(): vector_json = r""" { "name": "no_tool_calls", @@ -5838,7 +5866,7 @@ async def test_vector_100_turnconformance_run_no_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_101_turnconformance_run_parallel_tools_basic(): +async def test_vector_102_turnconformance_run_parallel_tools_basic(): vector_json = r""" { "name": "parallel_tools_basic", @@ -6011,7 +6039,7 @@ async def test_vector_101_turnconformance_run_parallel_tools_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_102_turnconformance_run_parallel_tools_with_guardrail_deny(): +async def test_vector_103_turnconformance_run_parallel_tools_with_guardrail_deny(): vector_json = r""" { "name": "parallel_tools_with_guardrail_deny", @@ -6202,7 +6230,7 @@ async def test_vector_102_turnconformance_run_parallel_tools_with_guardrail_deny await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_103_turnconformance_run_parallel_tools_with_tool_error(): +async def test_vector_104_turnconformance_run_parallel_tools_with_tool_error(): vector_json = r""" { "name": "parallel_tools_with_tool_error", @@ -6375,7 +6403,7 @@ async def test_vector_103_turnconformance_run_parallel_tools_with_tool_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_104_turnconformance_run_single_tool_call(): +async def test_vector_105_turnconformance_run_single_tool_call(): vector_json = r""" { "name": "single_tool_call", @@ -6522,7 +6550,7 @@ async def test_vector_104_turnconformance_run_single_tool_call(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_105_turnconformance_run_steering_inject_message(): +async def test_vector_106_turnconformance_run_steering_inject_message(): vector_json = r""" { "name": "steering_inject_message", @@ -6730,7 +6758,7 @@ async def test_vector_105_turnconformance_run_steering_inject_message(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_106_turnconformance_run_steering_multiple_messages(): +async def test_vector_107_turnconformance_run_steering_multiple_messages(): vector_json = r""" { "name": "steering_multiple_messages", @@ -6900,7 +6928,7 @@ async def test_vector_106_turnconformance_run_steering_multiple_messages(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_107_turnconformance_run_tool_not_registered_error(): +async def test_vector_108_turnconformance_run_tool_not_registered_error(): vector_json = r""" { "name": "tool_not_registered_error", @@ -6984,7 +7012,7 @@ async def test_vector_107_turnconformance_run_tool_not_registered_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_108_turnconformance_run_tool_result_message_format(): +async def test_vector_109_turnconformance_run_tool_result_message_format(): vector_json = r""" { "name": "tool_result_message_format", @@ -7105,7 +7133,7 @@ async def test_vector_108_turnconformance_run_tool_result_message_format(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_109_turnconformance_runturn_cancel_before_context(): +async def test_vector_110_turnconformance_runturn_cancel_before_context(): vector_json = r""" { "name": "cancel_before_context", @@ -7137,7 +7165,7 @@ async def test_vector_109_turnconformance_runturn_cancel_before_context(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_110_turnconformance_runturn_delegated_provider_state(): +async def test_vector_111_turnconformance_runturn_delegated_provider_state(): vector_json = r""" { "name": "delegated_provider_state", @@ -7204,7 +7232,7 @@ async def test_vector_110_turnconformance_runturn_delegated_provider_state(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_111_turnconformance_runturn_final_output(): +async def test_vector_112_turnconformance_runturn_final_output(): vector_json = r""" { "name": "final_output", @@ -7249,7 +7277,7 @@ async def test_vector_111_turnconformance_runturn_final_output(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_112_turnconformance_runturn_memory_context_is_model_visible(): +async def test_vector_113_turnconformance_runturn_memory_context_is_model_visible(): vector_json = r""" { "name": "memory_context_is_model_visible", @@ -7311,7 +7339,7 @@ async def test_vector_112_turnconformance_runturn_memory_context_is_model_visibl await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_113_turnconformance_runturn_ordered_tool_round(): +async def test_vector_114_turnconformance_runturn_ordered_tool_round(): vector_json = r""" { "name": "ordered_tool_round", @@ -7402,7 +7430,7 @@ async def test_vector_113_turnconformance_runturn_ordered_tool_round(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_114_turnconformance_runturn_permission_denial_is_model_visible(): +async def test_vector_115_turnconformance_runturn_permission_denial_is_model_visible(): vector_json = r""" { "name": "permission_denial_is_model_visible", @@ -7453,7 +7481,7 @@ async def test_vector_114_turnconformance_runturn_permission_denial_is_model_vis await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_115_wireconformance_torequest_anthropic_image_format(): +async def test_vector_116_wireconformance_torequest_anthropic_image_format(): vector_json = r""" { "name": "anthropic_image_format", @@ -7523,7 +7551,7 @@ async def test_vector_115_wireconformance_torequest_anthropic_image_format(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_116_wireconformance_torequest_anthropic_max_tokens_required(): +async def test_vector_117_wireconformance_torequest_anthropic_max_tokens_required(): vector_json = r""" { "name": "anthropic_max_tokens_required", @@ -7578,7 +7606,7 @@ async def test_vector_116_wireconformance_torequest_anthropic_max_tokens_require await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_117_wireconformance_torequest_anthropic_options(): +async def test_vector_118_wireconformance_torequest_anthropic_options(): vector_json = r""" { "name": "anthropic_options", @@ -7647,7 +7675,7 @@ async def test_vector_117_wireconformance_torequest_anthropic_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_118_wireconformance_torequest_anthropic_system_separate(): +async def test_vector_119_wireconformance_torequest_anthropic_system_separate(): vector_json = r""" { "name": "anthropic_system_separate", @@ -7714,7 +7742,7 @@ async def test_vector_118_wireconformance_torequest_anthropic_system_separate(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_119_wireconformance_torequest_anthropic_tool_wire(): +async def test_vector_120_wireconformance_torequest_anthropic_tool_wire(): vector_json = r""" { "name": "anthropic_tool_wire", @@ -7799,7 +7827,7 @@ async def test_vector_119_wireconformance_torequest_anthropic_tool_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_120_wireconformance_torequest_chat_audio_mp3(): +async def test_vector_121_wireconformance_torequest_chat_audio_mp3(): vector_json = r""" { "name": "chat_audio_mp3", @@ -7857,7 +7885,7 @@ async def test_vector_120_wireconformance_torequest_chat_audio_mp3(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_121_wireconformance_torequest_chat_audio_part(): +async def test_vector_122_wireconformance_torequest_chat_audio_part(): vector_json = r""" { "name": "chat_audio_part", @@ -7915,7 +7943,7 @@ async def test_vector_121_wireconformance_torequest_chat_audio_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_122_wireconformance_torequest_chat_image_base64(): +async def test_vector_123_wireconformance_torequest_chat_image_base64(): vector_json = r""" { "name": "chat_image_base64", @@ -7979,7 +8007,7 @@ async def test_vector_122_wireconformance_torequest_chat_image_base64(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_123_wireconformance_torequest_chat_image_part(): +async def test_vector_124_wireconformance_torequest_chat_image_part(): vector_json = r""" { "name": "chat_image_part", @@ -8043,7 +8071,7 @@ async def test_vector_123_wireconformance_torequest_chat_image_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_124_wireconformance_torequest_chat_multipart_content(): +async def test_vector_125_wireconformance_torequest_chat_multipart_content(): vector_json = r""" { "name": "chat_multipart_content", @@ -8107,7 +8135,7 @@ async def test_vector_124_wireconformance_torequest_chat_multipart_content(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_125_wireconformance_torequest_chat_simple(): +async def test_vector_126_wireconformance_torequest_chat_simple(): vector_json = r""" { "name": "chat_simple", @@ -8169,7 +8197,7 @@ async def test_vector_125_wireconformance_torequest_chat_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_126_wireconformance_torequest_chat_single_text_optimized(): +async def test_vector_127_wireconformance_torequest_chat_single_text_optimized(): vector_json = r""" { "name": "chat_single_text_optimized", @@ -8218,7 +8246,7 @@ async def test_vector_126_wireconformance_torequest_chat_single_text_optimized() await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_127_wireconformance_torequest_chat_with_options(): +async def test_vector_128_wireconformance_torequest_chat_with_options(): vector_json = r""" { "name": "chat_with_options", @@ -8286,7 +8314,7 @@ async def test_vector_127_wireconformance_torequest_chat_with_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_128_wireconformance_torequest_embedding_wire(): +async def test_vector_129_wireconformance_torequest_embedding_wire(): vector_json = r""" { "name": "embedding_wire", @@ -8330,7 +8358,7 @@ async def test_vector_128_wireconformance_torequest_embedding_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_129_wireconformance_torequest_image_wire(): +async def test_vector_130_wireconformance_torequest_image_wire(): vector_json = r""" { "name": "image_wire", @@ -8374,7 +8402,7 @@ async def test_vector_129_wireconformance_torequest_image_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_130_wireconformance_torequest_kind_to_json_type_mapping(): +async def test_vector_131_wireconformance_torequest_kind_to_json_type_mapping(): vector_json = r""" { "name": "kind_to_json_type_mapping", @@ -8493,7 +8521,7 @@ async def test_vector_130_wireconformance_torequest_kind_to_json_type_mapping(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_131_wireconformance_torequest_options_additional_properties(): +async def test_vector_132_wireconformance_torequest_options_additional_properties(): vector_json = r""" { "name": "options_additional_properties", @@ -8547,7 +8575,7 @@ async def test_vector_131_wireconformance_torequest_options_additional_propertie await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_132_wireconformance_torequest_options_max_completion_tokens(): +async def test_vector_133_wireconformance_torequest_options_max_completion_tokens(): vector_json = r""" { "name": "options_max_completion_tokens", @@ -8599,7 +8627,7 @@ async def test_vector_132_wireconformance_torequest_options_max_completion_token await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_133_wireconformance_torequest_options_stop_sequences(): +async def test_vector_134_wireconformance_torequest_options_stop_sequences(): vector_json = r""" { "name": "options_stop_sequences", @@ -8657,7 +8685,7 @@ async def test_vector_133_wireconformance_torequest_options_stop_sequences(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_134_wireconformance_torequest_responses_simple(): +async def test_vector_135_wireconformance_torequest_responses_simple(): vector_json = r""" { "name": "responses_simple", @@ -8716,7 +8744,7 @@ async def test_vector_134_wireconformance_torequest_responses_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_135_wireconformance_torequest_responses_structured_output(): +async def test_vector_136_wireconformance_torequest_responses_structured_output(): vector_json = r""" { "name": "responses_structured_output", @@ -8803,7 +8831,7 @@ async def test_vector_135_wireconformance_torequest_responses_structured_output( await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_136_wireconformance_torequest_responses_with_tools(): +async def test_vector_137_wireconformance_torequest_responses_with_tools(): vector_json = r""" { "name": "responses_with_tools", @@ -8883,7 +8911,7 @@ async def test_vector_136_wireconformance_torequest_responses_with_tools(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_137_wireconformance_torequest_structured_output(): +async def test_vector_138_wireconformance_torequest_structured_output(): vector_json = r""" { "name": "structured_output", @@ -8966,7 +8994,7 @@ async def test_vector_137_wireconformance_torequest_structured_output(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_138_wireconformance_torequest_structured_output_nested_optional(): +async def test_vector_139_wireconformance_torequest_structured_output_nested_optional(): vector_json = r""" { "name": "structured_output_nested_optional", @@ -9068,7 +9096,7 @@ async def test_vector_138_wireconformance_torequest_structured_output_nested_opt await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_139_wireconformance_torequest_tools_bindings_stripped(): +async def test_vector_140_wireconformance_torequest_tools_bindings_stripped(): vector_json = r""" { "name": "tools_bindings_stripped", @@ -9160,7 +9188,7 @@ async def test_vector_139_wireconformance_torequest_tools_bindings_stripped(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_140_wireconformance_torequest_tools_function_wire(): +async def test_vector_141_wireconformance_torequest_tools_function_wire(): vector_json = r""" { "name": "tools_function_wire", @@ -9242,7 +9270,7 @@ async def test_vector_140_wireconformance_torequest_tools_function_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_141_wireconformance_torequest_tools_null_when_empty(): +async def test_vector_142_wireconformance_torequest_tools_null_when_empty(): vector_json = r""" { "name": "tools_null_when_empty", @@ -9291,7 +9319,7 @@ async def test_vector_141_wireconformance_torequest_tools_null_when_empty(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_142_wireconformance_torequest_tools_strict_mode(): +async def test_vector_143_wireconformance_torequest_tools_strict_mode(): vector_json = r""" { "name": "tools_strict_mode", diff --git a/runtime/python/prompty/tests/test_endpoint_trust.py b/runtime/python/prompty/tests/test_endpoint_trust.py index 268e11a6a..c4517c86b 100644 --- a/runtime/python/prompty/tests/test_endpoint_trust.py +++ b/runtime/python/prompty/tests/test_endpoint_trust.py @@ -46,6 +46,7 @@ def test_builtin_trusted(self, url: str): "https://api.openai.com.evil.com", "https://evilopenai.azure.com", "https://api.openai.com@evil.com", # userinfo ignored; host is evil.com + "https://evil.com\\@api.openai.com", # backslash: clients connect to evil.com "http://localhost:11434", "not a url", "ftp://api.openai.com", diff --git a/runtime/rust/prompty/src/model/vector_conformance.rs b/runtime/rust/prompty/src/model/vector_conformance.rs index f33b82bbd..331462f72 100644 --- a/runtime/rust/prompty/src/model/vector_conformance.rs +++ b/runtime/rust/prompty/src/model/vector_conformance.rs @@ -1555,6 +1555,20 @@ pub async fn run_load_conformance_conformance Origin? { + // HTTP clients treat '\' as '/', so parsers disagree on the host; reject outright. + if url.contains("\\") { return nil } let trimmed = url.trimmingCharacters(in: .whitespacesAndNewlines) let text = String(String.UnicodeScalarView(trimmed.unicodeScalars.filter { !"\t\r\n".unicodeScalars.contains($0) })) guard let colon = text.firstIndex(of: ":") else { return nil } diff --git a/runtime/swift/prompty-model/Sources/PromptyModel/vector_conformance.swift b/runtime/swift/prompty-model/Sources/PromptyModel/vector_conformance.swift index 40a37163c..f833c6290 100644 --- a/runtime/swift/prompty-model/Sources/PromptyModel/vector_conformance.swift +++ b/runtime/swift/prompty-model/Sources/PromptyModel/vector_conformance.swift @@ -771,6 +771,23 @@ public func runLoadConformanceConformance(_ seam: S) async t throw SeamConformanceError("untrusted_endpoint_ambient_rejected: expected an error") } } + // vector: untrusted_endpoint_backslash_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_backslash_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_backslash_rejected: expected an error") + } + } // vector: untrusted_endpoint_env_default_rejected do { let inputData = "{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}}".data(using: .utf8)! diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift index 3e00f1af8..7e65efd4e 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift @@ -584,7 +584,17 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector55LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() async throws { + func testVector55LoadConformanceLoadUntrustedEndpointBackslashRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_backslash_rejected\",\"description\":\"A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -594,7 +604,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector56LoadConformanceLoadUntrustedEndpointEnvKeyRejected() async throws { + func testVector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -604,7 +614,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector57LoadConformanceLoadUntrustedEndpointFileConnectionRejected() async throws { + func testVector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -614,7 +624,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector58LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() async throws { + func testVector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -624,7 +634,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector59LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() async throws { + func testVector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -634,7 +644,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector60LoadConformanceLoadUntrustedEndpointNoKeyRejected() async throws { + func testVector61LoadConformanceLoadUntrustedEndpointNoKeyRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -644,7 +654,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector61LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() async throws { + func testVector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -654,7 +664,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector62LoadConformanceLoadUntrustedEndpointWildcardApexRejected() async throws { + func testVector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -664,7 +674,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector63MemoryConformanceOperateClearByCategoryAndAll() async throws { + func testVector64MemoryConformanceOperateClearByCategoryAndAll() async throws { guard let vectorData = "{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -674,7 +684,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector64MemoryConformanceOperateFormatCoreAndRecallResults() async throws { + func testVector65MemoryConformanceOperateFormatCoreAndRecallResults() async throws { guard let vectorData = "{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -684,7 +694,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector65MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { + func testVector66MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { guard let vectorData = "{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -694,7 +704,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector66MemoryConformanceOperateRecallRanksByWeightedScore() async throws { + func testVector67MemoryConformanceOperateRecallRanksByWeightedScore() async throws { guard let vectorData = "{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -704,7 +714,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector67MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { + func testVector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { guard let vectorData = "{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -714,7 +724,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector68MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { + func testVector69MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { guard let vectorData = "{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -724,7 +734,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector69MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { + func testVector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { guard let vectorData = "{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\u{e9}\"]},{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\u{e9}\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -734,7 +744,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector70MemoryConformanceOperateRememberDedupsCoreByTags() async throws { + func testVector71MemoryConformanceOperateRememberDedupsCoreByTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -744,7 +754,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector71MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { + func testVector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -754,7 +764,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector72MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { + func testVector73MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { guard let vectorData = "{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -764,7 +774,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector73MemoryConformanceOperateSnapshotPortRoundTrip() async throws { + func testVector74MemoryConformanceOperateSnapshotPortRoundTrip() async throws { guard let vectorData = "{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -774,7 +784,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector74MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { + func testVector75MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { guard let vectorData = "{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -784,7 +794,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector75TurnConformanceReplayMaxIterations() async throws { + func testVector76TurnConformanceReplayMaxIterations() async throws { guard let vectorData = "{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -794,7 +804,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector76TurnConformanceReplayNoTool() async throws { + func testVector77TurnConformanceReplayNoTool() async throws { guard let vectorData = "{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -804,7 +814,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector77TurnConformanceReplayPermissionDenied() async throws { + func testVector78TurnConformanceReplayPermissionDenied() async throws { guard let vectorData = "{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -814,7 +824,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector78TurnConformanceReplayToolFailure() async throws { + func testVector79TurnConformanceReplayToolFailure() async throws { guard let vectorData = "{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -824,7 +834,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector79TurnConformanceReplayToolSuccess() async throws { + func testVector80TurnConformanceReplayToolSuccess() async throws { guard let vectorData = "{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -834,7 +844,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector80TurnConformanceRunAssistantToolCallsMetadata() async throws { + func testVector81TurnConformanceRunAssistantToolCallsMetadata() async throws { guard let vectorData = "{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -844,7 +854,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector81TurnConformanceRunAsyncToolFunction() async throws { + func testVector82TurnConformanceRunAsyncToolFunction() async throws { guard let vectorData = "{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -854,7 +864,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector82TurnConformanceRunBindingsInjected() async throws { + func testVector83TurnConformanceRunBindingsInjected() async throws { guard let vectorData = "{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -864,7 +874,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector83TurnConformanceRunCancellationBeforeLlm() async throws { + func testVector84TurnConformanceRunCancellationBeforeLlm() async throws { guard let vectorData = "{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -874,7 +884,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector84TurnConformanceRunCancellationBetweenIterations() async throws { + func testVector85TurnConformanceRunCancellationBetweenIterations() async throws { guard let vectorData = "{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -884,7 +894,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector85TurnConformanceRunCancellationBetweenTools() async throws { + func testVector86TurnConformanceRunCancellationBetweenTools() async throws { guard let vectorData = "{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -894,7 +904,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector86TurnConformanceRunContextNoTrimWhenFits() async throws { + func testVector87TurnConformanceRunContextNoTrimWhenFits() async throws { guard let vectorData = "{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -904,7 +914,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector87TurnConformanceRunContextPreservesSystemMessages() async throws { + func testVector88TurnConformanceRunContextPreservesSystemMessages() async throws { guard let vectorData = "{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -914,7 +924,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector88TurnConformanceRunContextTrimBasic() async throws { + func testVector89TurnConformanceRunContextTrimBasic() async throws { guard let vectorData = "{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -924,7 +934,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector89TurnConformanceRunEmptyToolResult() async throws { + func testVector90TurnConformanceRunEmptyToolResult() async throws { guard let vectorData = "{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -934,7 +944,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector90TurnConformanceRunEventsBasicToolLoop() async throws { + func testVector91TurnConformanceRunEventsBasicToolLoop() async throws { guard let vectorData = "{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -944,7 +954,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector91TurnConformanceRunEventsErrorLogged() async throws { + func testVector92TurnConformanceRunEventsErrorLogged() async throws { guard let vectorData = "{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -954,7 +964,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector92TurnConformanceRunEventsNoTools() async throws { + func testVector93TurnConformanceRunEventsNoTools() async throws { guard let vectorData = "{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -964,7 +974,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector93TurnConformanceRunGuardrailAllPass() async throws { + func testVector94TurnConformanceRunGuardrailAllPass() async throws { guard let vectorData = "{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -974,7 +984,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector94TurnConformanceRunGuardrailInputDeny() async throws { + func testVector95TurnConformanceRunGuardrailInputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -984,7 +994,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector95TurnConformanceRunGuardrailOutputDeny() async throws { + func testVector96TurnConformanceRunGuardrailOutputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -994,7 +1004,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector96TurnConformanceRunGuardrailToolDeny() async throws { + func testVector97TurnConformanceRunGuardrailToolDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1004,7 +1014,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector97TurnConformanceRunMaxIterationsExceeded() async throws { + func testVector98TurnConformanceRunMaxIterationsExceeded() async throws { guard let vectorData = "{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1014,7 +1024,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector98TurnConformanceRunMultiTurnToolCalls() async throws { + func testVector99TurnConformanceRunMultiTurnToolCalls() async throws { guard let vectorData = "{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1024,7 +1034,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector99TurnConformanceRunMultipleToolCallsSingleTurn() async throws { + func testVector100TurnConformanceRunMultipleToolCallsSingleTurn() async throws { guard let vectorData = "{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1034,7 +1044,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector100TurnConformanceRunNoToolCalls() async throws { + func testVector101TurnConformanceRunNoToolCalls() async throws { guard let vectorData = "{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1044,7 +1054,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector101TurnConformanceRunParallelToolsBasic() async throws { + func testVector102TurnConformanceRunParallelToolsBasic() async throws { guard let vectorData = "{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1054,7 +1064,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector102TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { + func testVector103TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1064,7 +1074,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector103TurnConformanceRunParallelToolsWithToolError() async throws { + func testVector104TurnConformanceRunParallelToolsWithToolError() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\u{a7}13.6 Parallel Tools + tool error \u{2014} 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1074,7 +1084,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector104TurnConformanceRunSingleToolCall() async throws { + func testVector105TurnConformanceRunSingleToolCall() async throws { guard let vectorData = "{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1084,7 +1094,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector105TurnConformanceRunSteeringInjectMessage() async throws { + func testVector106TurnConformanceRunSteeringInjectMessage() async throws { guard let vectorData = "{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1094,7 +1104,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector106TurnConformanceRunSteeringMultipleMessages() async throws { + func testVector107TurnConformanceRunSteeringMultipleMessages() async throws { guard let vectorData = "{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1104,7 +1114,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector107TurnConformanceRunToolNotRegisteredError() async throws { + func testVector108TurnConformanceRunToolNotRegisteredError() async throws { guard let vectorData = "{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1114,7 +1124,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector108TurnConformanceRunToolResultMessageFormat() async throws { + func testVector109TurnConformanceRunToolResultMessageFormat() async throws { guard let vectorData = "{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1124,7 +1134,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector109TurnConformanceRunTurnCancelBeforeContext() async throws { + func testVector110TurnConformanceRunTurnCancelBeforeContext() async throws { guard let vectorData = "{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1134,7 +1144,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector110TurnConformanceRunTurnDelegatedProviderState() async throws { + func testVector111TurnConformanceRunTurnDelegatedProviderState() async throws { guard let vectorData = "{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1144,7 +1154,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector111TurnConformanceRunTurnFinalOutput() async throws { + func testVector112TurnConformanceRunTurnFinalOutput() async throws { guard let vectorData = "{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1154,7 +1164,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector112TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { + func testVector113TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { guard let vectorData = "{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1164,7 +1174,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector113TurnConformanceRunTurnOrderedToolRound() async throws { + func testVector114TurnConformanceRunTurnOrderedToolRound() async throws { guard let vectorData = "{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1174,7 +1184,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector114TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { + func testVector115TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { guard let vectorData = "{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1184,7 +1194,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector115WireConformanceToRequestAnthropicImageFormat() async throws { + func testVector116WireConformanceToRequestAnthropicImageFormat() async throws { guard let vectorData = "{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1194,7 +1204,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector116WireConformanceToRequestAnthropicMaxTokensRequired() async throws { + func testVector117WireConformanceToRequestAnthropicMaxTokensRequired() async throws { guard let vectorData = "{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1204,7 +1214,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector117WireConformanceToRequestAnthropicOptions() async throws { + func testVector118WireConformanceToRequestAnthropicOptions() async throws { guard let vectorData = "{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1214,7 +1224,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector118WireConformanceToRequestAnthropicSystemSeparate() async throws { + func testVector119WireConformanceToRequestAnthropicSystemSeparate() async throws { guard let vectorData = "{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1224,7 +1234,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector119WireConformanceToRequestAnthropicToolWire() async throws { + func testVector120WireConformanceToRequestAnthropicToolWire() async throws { guard let vectorData = "{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1234,7 +1244,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector120WireConformanceToRequestChatAudioMp3() async throws { + func testVector121WireConformanceToRequestChatAudioMp3() async throws { guard let vectorData = "{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1244,7 +1254,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector121WireConformanceToRequestChatAudioPart() async throws { + func testVector122WireConformanceToRequestChatAudioPart() async throws { guard let vectorData = "{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1254,7 +1264,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector122WireConformanceToRequestChatImageBase64() async throws { + func testVector123WireConformanceToRequestChatImageBase64() async throws { guard let vectorData = "{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1264,7 +1274,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector123WireConformanceToRequestChatImagePart() async throws { + func testVector124WireConformanceToRequestChatImagePart() async throws { guard let vectorData = "{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1274,7 +1284,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector124WireConformanceToRequestChatMultipartContent() async throws { + func testVector125WireConformanceToRequestChatMultipartContent() async throws { guard let vectorData = "{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1284,7 +1294,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector125WireConformanceToRequestChatSimple() async throws { + func testVector126WireConformanceToRequestChatSimple() async throws { guard let vectorData = "{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1294,7 +1304,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector126WireConformanceToRequestChatSingleTextOptimized() async throws { + func testVector127WireConformanceToRequestChatSingleTextOptimized() async throws { guard let vectorData = "{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1304,7 +1314,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector127WireConformanceToRequestChatWithOptions() async throws { + func testVector128WireConformanceToRequestChatWithOptions() async throws { guard let vectorData = "{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1314,7 +1324,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector128WireConformanceToRequestEmbeddingWire() async throws { + func testVector129WireConformanceToRequestEmbeddingWire() async throws { guard let vectorData = "{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1324,7 +1334,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector129WireConformanceToRequestImageWire() async throws { + func testVector130WireConformanceToRequestImageWire() async throws { guard let vectorData = "{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1334,7 +1344,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector130WireConformanceToRequestKindToJsonTypeMapping() async throws { + func testVector131WireConformanceToRequestKindToJsonTypeMapping() async throws { guard let vectorData = "{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1344,7 +1354,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector131WireConformanceToRequestOptionsAdditionalProperties() async throws { + func testVector132WireConformanceToRequestOptionsAdditionalProperties() async throws { guard let vectorData = "{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1354,7 +1364,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector132WireConformanceToRequestOptionsMaxCompletionTokens() async throws { + func testVector133WireConformanceToRequestOptionsMaxCompletionTokens() async throws { guard let vectorData = "{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1364,7 +1374,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector133WireConformanceToRequestOptionsStopSequences() async throws { + func testVector134WireConformanceToRequestOptionsStopSequences() async throws { guard let vectorData = "{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1374,7 +1384,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector134WireConformanceToRequestResponsesSimple() async throws { + func testVector135WireConformanceToRequestResponsesSimple() async throws { guard let vectorData = "{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1384,7 +1394,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector135WireConformanceToRequestResponsesStructuredOutput() async throws { + func testVector136WireConformanceToRequestResponsesStructuredOutput() async throws { guard let vectorData = "{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1394,7 +1404,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector136WireConformanceToRequestResponsesWithTools() async throws { + func testVector137WireConformanceToRequestResponsesWithTools() async throws { guard let vectorData = "{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1404,7 +1414,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector137WireConformanceToRequestStructuredOutput() async throws { + func testVector138WireConformanceToRequestStructuredOutput() async throws { guard let vectorData = "{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1414,7 +1424,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector138WireConformanceToRequestStructuredOutputNestedOptional() async throws { + func testVector139WireConformanceToRequestStructuredOutputNestedOptional() async throws { guard let vectorData = "{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1424,7 +1434,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector139WireConformanceToRequestToolsBindingsStripped() async throws { + func testVector140WireConformanceToRequestToolsBindingsStripped() async throws { guard let vectorData = "{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1434,7 +1444,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector140WireConformanceToRequestToolsFunctionWire() async throws { + func testVector141WireConformanceToRequestToolsFunctionWire() async throws { guard let vectorData = "{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1444,7 +1454,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector141WireConformanceToRequestToolsNullWhenEmpty() async throws { + func testVector142WireConformanceToRequestToolsNullWhenEmpty() async throws { guard let vectorData = "{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1454,7 +1464,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector142WireConformanceToRequestToolsStrictMode() async throws { + func testVector143WireConformanceToRequestToolsStrictMode() async throws { guard let vectorData = "{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { diff --git a/runtime/typescript/packages/core/src/model/vector-conformance.ts b/runtime/typescript/packages/core/src/model/vector-conformance.ts index e0114591c..ce3d820ae 100644 --- a/runtime/typescript/packages/core/src/model/vector-conformance.ts +++ b/runtime/typescript/packages/core/src/model/vector-conformance.ts @@ -761,6 +761,21 @@ export async function runLoadConformanceConformance( "untrusted_endpoint_ambient_rejected: expected an error", ); } + // vector: untrusted_endpoint_backslash_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_backslash_rejected: expected an error", + ); + } // vector: untrusted_endpoint_env_default_rejected { const sourceJson = "null"; diff --git a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts index 256c997bc..4799b1531 100644 --- a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts +++ b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts @@ -369,6 +369,12 @@ describe("callable vector conformance", () => { ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:untrusted_endpoint_backslash_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_backslash_rejected","description":"A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted","stage":"load","input":{"frontmatter":{"name":"untrusted-backslash","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://evil.example.com\\\\@api.openai.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("LoadConformance.load:untrusted_endpoint_env_default_rejected", async () => { const vector = JSON.parse( '{"name":"untrusted_endpoint_env_default_rejected","description":"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-env-default","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"${env:NONEXISTENT:https://attacker.example.com}"}}},"env":{}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', diff --git a/schema/model/conformance/vectors/load.tsp b/schema/model/conformance/vectors/load.tsp index 7e6bbb825..b6d8a01db 100644 --- a/schema/model/conformance/vectors/load.tsp +++ b/schema/model/conformance/vectors/load.tsp @@ -377,6 +377,26 @@ const LoadVectors = """ "kind": "untrusted_endpoint" } }, + { + "name": "untrusted_endpoint_backslash_rejected", + "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-backslash", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com\\\\@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, { "name": "untrusted_endpoint_env_default_rejected", "description": "An \${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", diff --git a/schema/tsp-output/.typra-generated/vectors.json b/schema/tsp-output/.typra-generated/vectors.json index ac9d1e56f..c7ad8c695 100644 --- a/schema/tsp-output/.typra-generated/vectors.json +++ b/schema/tsp-output/.typra-generated/vectors.json @@ -2122,6 +2122,38 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_backslash_rejected", + "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-backslash", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com\\@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, { "contract": "LoadConformance", "namespace": "Prompty.Conformance.Seams", diff --git a/spec/spec.md b/spec/spec.md index 1cb0bb74f..fb86beb8a 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -326,7 +326,8 @@ Built-in: `https://api.openai.com`, `https://api.anthropic.com`, case-insensitive; default ports (`http`:80, `https`:443) are normalized; path, query, and userinfo are ignored. An entry `scheme://*.domain[:port]` matches any strict subdomain of `domain` (not `domain` itself). The entry `*` trusts every endpoint. -A destination that does not parse as an absolute `http`/`https` URL is untrusted. +A destination that does not parse as an absolute `http`/`https` URL, or that contains a +backslash (`\`), is untrusted. `.prompty` frontmatter MUST NOT be able to grant itself trust. Tool connections (§2.9) are out of scope today because no executor attaches ambient From ab4c33b2b0780ad14e7b4f279db5fb80588d41f0 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 17:53:39 -0700 Subject: [PATCH 11/15] test(spec): cover more endpoint trust URL forms Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Model/VectorConformanceTests.cs | 590 +++++++++++++++--- .../Prompty.Core/Model/VectorConformance.cs | 168 +++++ .../prompty/model/vector_conformance_test.go | 577 ++++++++++++++--- .../vectorconformance/vector_conformance.go | 173 +++++ .../prompty/model/VectorConformance.java | 180 ++++++ .../prompty/model/VectorConformanceTests.java | 484 ++++++++------ .../prompty/model/vector_conformance.py | 128 ++++ .../tests/model/test_vector_conformance.py | 564 ++++++++++++++--- .../prompty/src/model/vector_conformance.rs | 216 +++++++ .../tests/model/vector_conformance_test.rs | 281 ++++++--- .../PromptyModel/vector_conformance.swift | 215 +++++++ .../VectorConformanceTests.swift | 320 +++++++--- .../core/src/model/vector-conformance.ts | 193 ++++++ .../tests/model/vector-conformance.test.ts | 78 +++ schema/model/conformance/vectors/load.tsp | 270 ++++++++ .../tsp-output/.typra-generated/vectors.json | 426 +++++++++++++ spec/spec.md | 4 +- 17 files changed, 4201 insertions(+), 666 deletions(-) diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs index fdb644aee..53727818a 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs @@ -1832,7 +1832,42 @@ public async Task Vector48LoadConformanceLoadTrustedEndpointBuiltinAllowed() } [Fact] - public async Task Vector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() + public async Task Vector49LoadConformanceLoadTrustedEndpointDefaultPortAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_default_port_allowed", + "description": "An explicit default port normalizes to the built-in origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-default-port-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector50LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() { string vectorJson = """ { @@ -1870,7 +1905,7 @@ public async Task Vector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() } [Fact] - public async Task Vector50LoadConformanceLoadTrustedEndpointEnvListAllowed() + public async Task Vector51LoadConformanceLoadTrustedEndpointEnvListAllowed() { string vectorJson = """ { @@ -1908,7 +1943,7 @@ public async Task Vector50LoadConformanceLoadTrustedEndpointEnvListAllowed() } [Fact] - public async Task Vector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() + public async Task Vector52LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() { string vectorJson = """ { @@ -1944,7 +1979,7 @@ public async Task Vector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() } [Fact] - public async Task Vector52LoadConformanceLoadTrustedEndpointOptionAllowed() + public async Task Vector53LoadConformanceLoadTrustedEndpointOptionAllowed() { string vectorJson = """ { @@ -1982,7 +2017,7 @@ public async Task Vector52LoadConformanceLoadTrustedEndpointOptionAllowed() } [Fact] - public async Task Vector53LoadConformanceLoadTrustedEndpointReferenceSkipped() + public async Task Vector54LoadConformanceLoadTrustedEndpointReferenceSkipped() { string vectorJson = """ { @@ -2017,7 +2052,42 @@ public async Task Vector53LoadConformanceLoadTrustedEndpointReferenceSkipped() } [Fact] - public async Task Vector54LoadConformanceLoadUntrustedEndpointAmbientRejected() + public async Task Vector55LoadConformanceLoadTrustedEndpointUserinfoAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_userinfo_allowed", + "description": "Userinfo is ignored; the destination host api.openai.com is trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-userinfo-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector56LoadConformanceLoadUntrustedEndpointAmbientRejected() { string vectorJson = """ { @@ -2048,7 +2118,7 @@ public async Task Vector54LoadConformanceLoadUntrustedEndpointAmbientRejected() } [Fact] - public async Task Vector55LoadConformanceLoadUntrustedEndpointBackslashRejected() + public async Task Vector57LoadConformanceLoadUntrustedEndpointBackslashRejected() { string vectorJson = """ { @@ -2078,7 +2148,7 @@ public async Task Vector55LoadConformanceLoadUntrustedEndpointBackslashRejected( } [Fact] - public async Task Vector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() + public async Task Vector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() { string vectorJson = """ { @@ -2109,7 +2179,7 @@ public async Task Vector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected } [Fact] - public async Task Vector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected() + public async Task Vector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected() { string vectorJson = """ { @@ -2143,7 +2213,7 @@ public async Task Vector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected() } [Fact] - public async Task Vector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected() + public async Task Vector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected() { string vectorJson = """ { @@ -2176,7 +2246,67 @@ public async Task Vector58LoadConformanceLoadUntrustedEndpointFileConnectionReje } [Fact] - public async Task Vector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() + public async Task Vector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_fragment_before_at_rejected", + "description": "A '#' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-fragment-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com#@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_ipv6_literal_rejected", + "description": "An IPv6 literal is not a trusted origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ipv6-literal", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://[::1]:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() { string vectorJson = """ { @@ -2207,7 +2337,7 @@ public async Task Vector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecre } [Fact] - public async Task Vector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() + public async Task Vector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() { string vectorJson = """ { @@ -2238,7 +2368,7 @@ public async Task Vector60LoadConformanceLoadUntrustedEndpointMismatchedCredenti } [Fact] - public async Task Vector61LoadConformanceLoadUntrustedEndpointNoKeyRejected() + public async Task Vector65LoadConformanceLoadUntrustedEndpointNoKeyRejected() { string vectorJson = """ { @@ -2268,7 +2398,157 @@ public async Task Vector61LoadConformanceLoadUntrustedEndpointNoKeyRejected() } [Fact] - public async Task Vector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() + public async Task Vector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_percent_encoded_host_rejected", + "description": "Hosts are compared without percent-decoding, so an encoded host never matches", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-percent-encoded-host", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com%2eevil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_port_mismatch_rejected", + "description": "A non-default port is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-port-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com:8443" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_query_before_at_rejected", + "description": "A '?' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-query-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com?@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_scheme_mismatch_rejected", + "description": "Built-in origins are https; http on the same host is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-scheme-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_suffix_lookalike_rejected", + "description": "A trusted host as a label prefix of another domain is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-suffix-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com.evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() { string vectorJson = """ { @@ -2301,7 +2581,97 @@ public async Task Vector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejecte } [Fact] - public async Task Vector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected() + public async Task Vector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_trailing_dot_rejected", + "description": "Hosts are compared exactly; a trailing-dot host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-trailing-dot", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com." + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_unicode_homoglyph_rejected", + "description": "A non-ASCII lookalike host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-unicode-homoglyph", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://\u0430pi.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_userinfo_lookalike_rejected", + "description": "A trusted host in userinfo is not the destination host", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-userinfo-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com@evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected() { string vectorJson = """ { @@ -2331,7 +2701,37 @@ public async Task Vector63LoadConformanceLoadUntrustedEndpointWildcardApexReject } [Fact] - public async Task Vector64MemoryConformanceOperateClearByCategoryAndAll() + public async Task Vector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_wildcard_boundary_rejected", + "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-wildcard-boundary", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evilservices.ai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector77MemoryConformanceOperateClearByCategoryAndAll() { string vectorJson = """ { @@ -2385,7 +2785,7 @@ public async Task Vector64MemoryConformanceOperateClearByCategoryAndAll() } [Fact] - public async Task Vector65MemoryConformanceOperateFormatCoreAndRecallResults() + public async Task Vector78MemoryConformanceOperateFormatCoreAndRecallResults() { string vectorJson = """ { @@ -2427,7 +2827,7 @@ public async Task Vector65MemoryConformanceOperateFormatCoreAndRecallResults() } [Fact] - public async Task Vector66MemoryConformanceOperateRecallEmptyQueryAndLimit() + public async Task Vector79MemoryConformanceOperateRecallEmptyQueryAndLimit() { string vectorJson = """ { @@ -2482,7 +2882,7 @@ public async Task Vector66MemoryConformanceOperateRecallEmptyQueryAndLimit() } [Fact] - public async Task Vector67MemoryConformanceOperateRecallRanksByWeightedScore() + public async Task Vector80MemoryConformanceOperateRecallRanksByWeightedScore() { string vectorJson = """ { @@ -2541,7 +2941,7 @@ public async Task Vector67MemoryConformanceOperateRecallRanksByWeightedScore() } [Fact] - public async Task Vector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() + public async Task Vector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() { string vectorJson = """ { @@ -2599,7 +2999,7 @@ public async Task Vector68MemoryConformanceOperateRecallScoreTiesPreserveInserti } [Fact] - public async Task Vector69MemoryConformanceOperateRecallTagsAndCoreBoost() + public async Task Vector82MemoryConformanceOperateRecallTagsAndCoreBoost() { string vectorJson = """ { @@ -2660,7 +3060,7 @@ public async Task Vector69MemoryConformanceOperateRecallTagsAndCoreBoost() } [Fact] - public async Task Vector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() + public async Task Vector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() { string vectorJson = """ { @@ -2721,7 +3121,7 @@ public async Task Vector70MemoryConformanceOperateRecallTokenizationPunctuationU } [Fact] - public async Task Vector71MemoryConformanceOperateRememberDedupsCoreByTags() + public async Task Vector84MemoryConformanceOperateRememberDedupsCoreByTags() { string vectorJson = """ { @@ -2785,7 +3185,7 @@ public async Task Vector71MemoryConformanceOperateRememberDedupsCoreByTags() } [Fact] - public async Task Vector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() + public async Task Vector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() { string vectorJson = """ { @@ -2841,7 +3241,7 @@ public async Task Vector72MemoryConformanceOperateRememberDedupsCoreMissingAndEm } [Fact] - public async Task Vector73MemoryConformanceOperateRememberEvictsArchivalFirst() + public async Task Vector86MemoryConformanceOperateRememberEvictsArchivalFirst() { string vectorJson = """ { @@ -2902,7 +3302,7 @@ public async Task Vector73MemoryConformanceOperateRememberEvictsArchivalFirst() } [Fact] - public async Task Vector74MemoryConformanceOperateSnapshotPortRoundTrip() + public async Task Vector87MemoryConformanceOperateSnapshotPortRoundTrip() { string vectorJson = """ { @@ -2942,7 +3342,7 @@ public async Task Vector74MemoryConformanceOperateSnapshotPortRoundTrip() } [Fact] - public async Task Vector75MemoryConformanceOperateUpdateIndexOutOfRangeError() + public async Task Vector88MemoryConformanceOperateUpdateIndexOutOfRangeError() { string vectorJson = """ { @@ -2976,7 +3376,7 @@ public async Task Vector75MemoryConformanceOperateUpdateIndexOutOfRangeError() } [Fact] - public async Task Vector76TurnConformanceReplayMaxIterations() + public async Task Vector89TurnConformanceReplayMaxIterations() { string vectorJson = """ { @@ -3013,7 +3413,7 @@ public async Task Vector76TurnConformanceReplayMaxIterations() } [Fact] - public async Task Vector77TurnConformanceReplayNoTool() + public async Task Vector90TurnConformanceReplayNoTool() { string vectorJson = """ { @@ -3046,7 +3446,7 @@ public async Task Vector77TurnConformanceReplayNoTool() } [Fact] - public async Task Vector78TurnConformanceReplayPermissionDenied() + public async Task Vector91TurnConformanceReplayPermissionDenied() { string vectorJson = """ { @@ -3082,7 +3482,7 @@ public async Task Vector78TurnConformanceReplayPermissionDenied() } [Fact] - public async Task Vector79TurnConformanceReplayToolFailure() + public async Task Vector92TurnConformanceReplayToolFailure() { string vectorJson = """ { @@ -3120,7 +3520,7 @@ public async Task Vector79TurnConformanceReplayToolFailure() } [Fact] - public async Task Vector80TurnConformanceReplayToolSuccess() + public async Task Vector93TurnConformanceReplayToolSuccess() { string vectorJson = """ { @@ -3158,7 +3558,7 @@ public async Task Vector80TurnConformanceReplayToolSuccess() } [Fact] - public async Task Vector81TurnConformanceRunAssistantToolCallsMetadata() + public async Task Vector94TurnConformanceRunAssistantToolCallsMetadata() { string vectorJson = """ { @@ -3312,7 +3712,7 @@ public async Task Vector81TurnConformanceRunAssistantToolCallsMetadata() } [Fact] - public async Task Vector82TurnConformanceRunAsyncToolFunction() + public async Task Vector95TurnConformanceRunAsyncToolFunction() { string vectorJson = """ { @@ -3424,7 +3824,7 @@ public async Task Vector82TurnConformanceRunAsyncToolFunction() } [Fact] - public async Task Vector83TurnConformanceRunBindingsInjected() + public async Task Vector96TurnConformanceRunBindingsInjected() { string vectorJson = """ { @@ -3556,7 +3956,7 @@ public async Task Vector83TurnConformanceRunBindingsInjected() } [Fact] - public async Task Vector84TurnConformanceRunCancellationBeforeLlm() + public async Task Vector97TurnConformanceRunCancellationBeforeLlm() { string vectorJson = """ { @@ -3619,7 +4019,7 @@ public async Task Vector84TurnConformanceRunCancellationBeforeLlm() } [Fact] - public async Task Vector85TurnConformanceRunCancellationBetweenIterations() + public async Task Vector98TurnConformanceRunCancellationBetweenIterations() { string vectorJson = """ { @@ -3751,7 +4151,7 @@ public async Task Vector85TurnConformanceRunCancellationBetweenIterations() } [Fact] - public async Task Vector86TurnConformanceRunCancellationBetweenTools() + public async Task Vector99TurnConformanceRunCancellationBetweenTools() { string vectorJson = """ { @@ -3892,7 +4292,7 @@ public async Task Vector86TurnConformanceRunCancellationBetweenTools() } [Fact] - public async Task Vector87TurnConformanceRunContextNoTrimWhenFits() + public async Task Vector100TurnConformanceRunContextNoTrimWhenFits() { string vectorJson = """ { @@ -3965,7 +4365,7 @@ public async Task Vector87TurnConformanceRunContextNoTrimWhenFits() } [Fact] - public async Task Vector88TurnConformanceRunContextPreservesSystemMessages() + public async Task Vector101TurnConformanceRunContextPreservesSystemMessages() { string vectorJson = """ { @@ -4074,7 +4474,7 @@ public async Task Vector88TurnConformanceRunContextPreservesSystemMessages() } [Fact] - public async Task Vector89TurnConformanceRunContextTrimBasic() + public async Task Vector102TurnConformanceRunContextTrimBasic() { string vectorJson = """ { @@ -4234,7 +4634,7 @@ public async Task Vector89TurnConformanceRunContextTrimBasic() } [Fact] - public async Task Vector90TurnConformanceRunEmptyToolResult() + public async Task Vector103TurnConformanceRunEmptyToolResult() { string vectorJson = """ { @@ -4351,7 +4751,7 @@ public async Task Vector90TurnConformanceRunEmptyToolResult() } [Fact] - public async Task Vector91TurnConformanceRunEventsBasicToolLoop() + public async Task Vector104TurnConformanceRunEventsBasicToolLoop() { string vectorJson = """ { @@ -4499,7 +4899,7 @@ public async Task Vector91TurnConformanceRunEventsBasicToolLoop() } [Fact] - public async Task Vector92TurnConformanceRunEventsErrorLogged() + public async Task Vector105TurnConformanceRunEventsErrorLogged() { string vectorJson = """ { @@ -4627,7 +5027,7 @@ public async Task Vector92TurnConformanceRunEventsErrorLogged() } [Fact] - public async Task Vector93TurnConformanceRunEventsNoTools() + public async Task Vector106TurnConformanceRunEventsNoTools() { string vectorJson = """ { @@ -4707,7 +5107,7 @@ public async Task Vector93TurnConformanceRunEventsNoTools() } [Fact] - public async Task Vector94TurnConformanceRunGuardrailAllPass() + public async Task Vector107TurnConformanceRunGuardrailAllPass() { string vectorJson = """ { @@ -4832,7 +5232,7 @@ public async Task Vector94TurnConformanceRunGuardrailAllPass() } [Fact] - public async Task Vector95TurnConformanceRunGuardrailInputDeny() + public async Task Vector108TurnConformanceRunGuardrailInputDeny() { string vectorJson = """ { @@ -4890,7 +5290,7 @@ public async Task Vector95TurnConformanceRunGuardrailInputDeny() } [Fact] - public async Task Vector96TurnConformanceRunGuardrailOutputDeny() + public async Task Vector109TurnConformanceRunGuardrailOutputDeny() { string vectorJson = """ { @@ -4970,7 +5370,7 @@ public async Task Vector96TurnConformanceRunGuardrailOutputDeny() } [Fact] - public async Task Vector97TurnConformanceRunGuardrailToolDeny() + public async Task Vector110TurnConformanceRunGuardrailToolDeny() { string vectorJson = """ { @@ -5126,7 +5526,7 @@ public async Task Vector97TurnConformanceRunGuardrailToolDeny() } [Fact] - public async Task Vector98TurnConformanceRunMaxIterationsExceeded() + public async Task Vector111TurnConformanceRunMaxIterationsExceeded() { string vectorJson = """ { @@ -5638,7 +6038,7 @@ public async Task Vector98TurnConformanceRunMaxIterationsExceeded() } [Fact] - public async Task Vector99TurnConformanceRunMultiTurnToolCalls() + public async Task Vector112TurnConformanceRunMultiTurnToolCalls() { string vectorJson = """ { @@ -5817,7 +6217,7 @@ public async Task Vector99TurnConformanceRunMultiTurnToolCalls() } [Fact] - public async Task Vector100TurnConformanceRunMultipleToolCallsSingleTurn() + public async Task Vector113TurnConformanceRunMultipleToolCallsSingleTurn() { string vectorJson = """ { @@ -6000,7 +6400,7 @@ public async Task Vector100TurnConformanceRunMultipleToolCallsSingleTurn() } [Fact] - public async Task Vector101TurnConformanceRunNoToolCalls() + public async Task Vector114TurnConformanceRunNoToolCalls() { string vectorJson = """ { @@ -6070,7 +6470,7 @@ public async Task Vector101TurnConformanceRunNoToolCalls() } [Fact] - public async Task Vector102TurnConformanceRunParallelToolsBasic() + public async Task Vector115TurnConformanceRunParallelToolsBasic() { string vectorJson = """ { @@ -6245,7 +6645,7 @@ public async Task Vector102TurnConformanceRunParallelToolsBasic() } [Fact] - public async Task Vector103TurnConformanceRunParallelToolsWithGuardrailDeny() + public async Task Vector116TurnConformanceRunParallelToolsWithGuardrailDeny() { string vectorJson = """ { @@ -6438,7 +6838,7 @@ public async Task Vector103TurnConformanceRunParallelToolsWithGuardrailDeny() } [Fact] - public async Task Vector104TurnConformanceRunParallelToolsWithToolError() + public async Task Vector117TurnConformanceRunParallelToolsWithToolError() { string vectorJson = """ { @@ -6613,7 +7013,7 @@ public async Task Vector104TurnConformanceRunParallelToolsWithToolError() } [Fact] - public async Task Vector105TurnConformanceRunSingleToolCall() + public async Task Vector118TurnConformanceRunSingleToolCall() { string vectorJson = """ { @@ -6762,7 +7162,7 @@ public async Task Vector105TurnConformanceRunSingleToolCall() } [Fact] - public async Task Vector106TurnConformanceRunSteeringInjectMessage() + public async Task Vector119TurnConformanceRunSteeringInjectMessage() { string vectorJson = """ { @@ -6972,7 +7372,7 @@ public async Task Vector106TurnConformanceRunSteeringInjectMessage() } [Fact] - public async Task Vector107TurnConformanceRunSteeringMultipleMessages() + public async Task Vector120TurnConformanceRunSteeringMultipleMessages() { string vectorJson = """ { @@ -7144,7 +7544,7 @@ public async Task Vector107TurnConformanceRunSteeringMultipleMessages() } [Fact] - public async Task Vector108TurnConformanceRunToolNotRegisteredError() + public async Task Vector121TurnConformanceRunToolNotRegisteredError() { string vectorJson = """ { @@ -7230,7 +7630,7 @@ public async Task Vector108TurnConformanceRunToolNotRegisteredError() } [Fact] - public async Task Vector109TurnConformanceRunToolResultMessageFormat() + public async Task Vector122TurnConformanceRunToolResultMessageFormat() { string vectorJson = """ { @@ -7353,7 +7753,7 @@ public async Task Vector109TurnConformanceRunToolResultMessageFormat() } [Fact] - public async Task Vector110TurnConformanceRunTurnCancelBeforeContext() + public async Task Vector123TurnConformanceRunTurnCancelBeforeContext() { string vectorJson = """ { @@ -7387,7 +7787,7 @@ public async Task Vector110TurnConformanceRunTurnCancelBeforeContext() } [Fact] - public async Task Vector111TurnConformanceRunTurnDelegatedProviderState() + public async Task Vector124TurnConformanceRunTurnDelegatedProviderState() { string vectorJson = """ { @@ -7456,7 +7856,7 @@ public async Task Vector111TurnConformanceRunTurnDelegatedProviderState() } [Fact] - public async Task Vector112TurnConformanceRunTurnFinalOutput() + public async Task Vector125TurnConformanceRunTurnFinalOutput() { string vectorJson = """ { @@ -7503,7 +7903,7 @@ public async Task Vector112TurnConformanceRunTurnFinalOutput() } [Fact] - public async Task Vector113TurnConformanceRunTurnMemoryContextIsModelVisible() + public async Task Vector126TurnConformanceRunTurnMemoryContextIsModelVisible() { string vectorJson = """ { @@ -7567,7 +7967,7 @@ public async Task Vector113TurnConformanceRunTurnMemoryContextIsModelVisible() } [Fact] - public async Task Vector114TurnConformanceRunTurnOrderedToolRound() + public async Task Vector127TurnConformanceRunTurnOrderedToolRound() { string vectorJson = """ { @@ -7660,7 +8060,7 @@ public async Task Vector114TurnConformanceRunTurnOrderedToolRound() } [Fact] - public async Task Vector115TurnConformanceRunTurnPermissionDenialIsModelVisible() + public async Task Vector128TurnConformanceRunTurnPermissionDenialIsModelVisible() { string vectorJson = """ { @@ -7713,7 +8113,7 @@ public async Task Vector115TurnConformanceRunTurnPermissionDenialIsModelVisible( } [Fact] - public async Task Vector116WireConformanceToRequestAnthropicImageFormat() + public async Task Vector129WireConformanceToRequestAnthropicImageFormat() { string vectorJson = """ { @@ -7785,7 +8185,7 @@ public async Task Vector116WireConformanceToRequestAnthropicImageFormat() } [Fact] - public async Task Vector117WireConformanceToRequestAnthropicMaxTokensRequired() + public async Task Vector130WireConformanceToRequestAnthropicMaxTokensRequired() { string vectorJson = """ { @@ -7842,7 +8242,7 @@ public async Task Vector117WireConformanceToRequestAnthropicMaxTokensRequired() } [Fact] - public async Task Vector118WireConformanceToRequestAnthropicOptions() + public async Task Vector131WireConformanceToRequestAnthropicOptions() { string vectorJson = """ { @@ -7913,7 +8313,7 @@ public async Task Vector118WireConformanceToRequestAnthropicOptions() } [Fact] - public async Task Vector119WireConformanceToRequestAnthropicSystemSeparate() + public async Task Vector132WireConformanceToRequestAnthropicSystemSeparate() { string vectorJson = """ { @@ -7982,7 +8382,7 @@ public async Task Vector119WireConformanceToRequestAnthropicSystemSeparate() } [Fact] - public async Task Vector120WireConformanceToRequestAnthropicToolWire() + public async Task Vector133WireConformanceToRequestAnthropicToolWire() { string vectorJson = """ { @@ -8069,7 +8469,7 @@ public async Task Vector120WireConformanceToRequestAnthropicToolWire() } [Fact] - public async Task Vector121WireConformanceToRequestChatAudioMp3() + public async Task Vector134WireConformanceToRequestChatAudioMp3() { string vectorJson = """ { @@ -8129,7 +8529,7 @@ public async Task Vector121WireConformanceToRequestChatAudioMp3() } [Fact] - public async Task Vector122WireConformanceToRequestChatAudioPart() + public async Task Vector135WireConformanceToRequestChatAudioPart() { string vectorJson = """ { @@ -8189,7 +8589,7 @@ public async Task Vector122WireConformanceToRequestChatAudioPart() } [Fact] - public async Task Vector123WireConformanceToRequestChatImageBase64() + public async Task Vector136WireConformanceToRequestChatImageBase64() { string vectorJson = """ { @@ -8255,7 +8655,7 @@ public async Task Vector123WireConformanceToRequestChatImageBase64() } [Fact] - public async Task Vector124WireConformanceToRequestChatImagePart() + public async Task Vector137WireConformanceToRequestChatImagePart() { string vectorJson = """ { @@ -8321,7 +8721,7 @@ public async Task Vector124WireConformanceToRequestChatImagePart() } [Fact] - public async Task Vector125WireConformanceToRequestChatMultipartContent() + public async Task Vector138WireConformanceToRequestChatMultipartContent() { string vectorJson = """ { @@ -8387,7 +8787,7 @@ public async Task Vector125WireConformanceToRequestChatMultipartContent() } [Fact] - public async Task Vector126WireConformanceToRequestChatSimple() + public async Task Vector139WireConformanceToRequestChatSimple() { string vectorJson = """ { @@ -8451,7 +8851,7 @@ public async Task Vector126WireConformanceToRequestChatSimple() } [Fact] - public async Task Vector127WireConformanceToRequestChatSingleTextOptimized() + public async Task Vector140WireConformanceToRequestChatSingleTextOptimized() { string vectorJson = """ { @@ -8502,7 +8902,7 @@ public async Task Vector127WireConformanceToRequestChatSingleTextOptimized() } [Fact] - public async Task Vector128WireConformanceToRequestChatWithOptions() + public async Task Vector141WireConformanceToRequestChatWithOptions() { string vectorJson = """ { @@ -8572,7 +8972,7 @@ public async Task Vector128WireConformanceToRequestChatWithOptions() } [Fact] - public async Task Vector129WireConformanceToRequestEmbeddingWire() + public async Task Vector142WireConformanceToRequestEmbeddingWire() { string vectorJson = """ { @@ -8618,7 +9018,7 @@ public async Task Vector129WireConformanceToRequestEmbeddingWire() } [Fact] - public async Task Vector130WireConformanceToRequestImageWire() + public async Task Vector143WireConformanceToRequestImageWire() { string vectorJson = """ { @@ -8664,7 +9064,7 @@ public async Task Vector130WireConformanceToRequestImageWire() } [Fact] - public async Task Vector131WireConformanceToRequestKindToJsonTypeMapping() + public async Task Vector144WireConformanceToRequestKindToJsonTypeMapping() { string vectorJson = """ { @@ -8785,7 +9185,7 @@ public async Task Vector131WireConformanceToRequestKindToJsonTypeMapping() } [Fact] - public async Task Vector132WireConformanceToRequestOptionsAdditionalProperties() + public async Task Vector145WireConformanceToRequestOptionsAdditionalProperties() { string vectorJson = """ { @@ -8841,7 +9241,7 @@ public async Task Vector132WireConformanceToRequestOptionsAdditionalProperties() } [Fact] - public async Task Vector133WireConformanceToRequestOptionsMaxCompletionTokens() + public async Task Vector146WireConformanceToRequestOptionsMaxCompletionTokens() { string vectorJson = """ { @@ -8895,7 +9295,7 @@ public async Task Vector133WireConformanceToRequestOptionsMaxCompletionTokens() } [Fact] - public async Task Vector134WireConformanceToRequestOptionsStopSequences() + public async Task Vector147WireConformanceToRequestOptionsStopSequences() { string vectorJson = """ { @@ -8955,7 +9355,7 @@ public async Task Vector134WireConformanceToRequestOptionsStopSequences() } [Fact] - public async Task Vector135WireConformanceToRequestResponsesSimple() + public async Task Vector148WireConformanceToRequestResponsesSimple() { string vectorJson = """ { @@ -9016,7 +9416,7 @@ public async Task Vector135WireConformanceToRequestResponsesSimple() } [Fact] - public async Task Vector136WireConformanceToRequestResponsesStructuredOutput() + public async Task Vector149WireConformanceToRequestResponsesStructuredOutput() { string vectorJson = """ { @@ -9105,7 +9505,7 @@ public async Task Vector136WireConformanceToRequestResponsesStructuredOutput() } [Fact] - public async Task Vector137WireConformanceToRequestResponsesWithTools() + public async Task Vector150WireConformanceToRequestResponsesWithTools() { string vectorJson = """ { @@ -9187,7 +9587,7 @@ public async Task Vector137WireConformanceToRequestResponsesWithTools() } [Fact] - public async Task Vector138WireConformanceToRequestStructuredOutput() + public async Task Vector151WireConformanceToRequestStructuredOutput() { string vectorJson = """ { @@ -9272,7 +9672,7 @@ public async Task Vector138WireConformanceToRequestStructuredOutput() } [Fact] - public async Task Vector139WireConformanceToRequestStructuredOutputNestedOptional() + public async Task Vector152WireConformanceToRequestStructuredOutputNestedOptional() { string vectorJson = """ { @@ -9376,7 +9776,7 @@ public async Task Vector139WireConformanceToRequestStructuredOutputNestedOptiona } [Fact] - public async Task Vector140WireConformanceToRequestToolsBindingsStripped() + public async Task Vector153WireConformanceToRequestToolsBindingsStripped() { string vectorJson = """ { @@ -9470,7 +9870,7 @@ public async Task Vector140WireConformanceToRequestToolsBindingsStripped() } [Fact] - public async Task Vector141WireConformanceToRequestToolsFunctionWire() + public async Task Vector154WireConformanceToRequestToolsFunctionWire() { string vectorJson = """ { @@ -9554,7 +9954,7 @@ public async Task Vector141WireConformanceToRequestToolsFunctionWire() } [Fact] - public async Task Vector142WireConformanceToRequestToolsNullWhenEmpty() + public async Task Vector155WireConformanceToRequestToolsNullWhenEmpty() { string vectorJson = """ { @@ -9605,7 +10005,7 @@ public async Task Vector142WireConformanceToRequestToolsNullWhenEmpty() } [Fact] - public async Task Vector143WireConformanceToRequestToolsStrictMode() + public async Task Vector156WireConformanceToRequestToolsStrictMode() { string vectorJson = """ { diff --git a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs index eb38b13eb..7bb2958dd 100644 --- a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs +++ b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs @@ -394,6 +394,13 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}")) throw new SeamConformanceException("trusted_endpoint_builtin_allowed misrouted"); } + { + // vector: trusted_endpoint_default_port_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}")) + throw new SeamConformanceException("trusted_endpoint_default_port_allowed misrouted"); + } { // vector: trusted_endpoint_env_endpoint_allowed var source = JsonSerializer.Deserialize("null")!; @@ -429,6 +436,13 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}")) throw new SeamConformanceException("trusted_endpoint_reference_skipped misrouted"); } + { + // vector: trusted_endpoint_userinfo_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}")) + throw new SeamConformanceException("trusted_endpoint_userinfo_allowed misrouted"); + } { // vector: untrusted_endpoint_ambient_rejected var source = JsonSerializer.Deserialize("null")!; @@ -499,6 +513,34 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea } if (caught is null) throw new SeamConformanceException("untrusted_endpoint_file_connection_rejected: expected an error"); } + { + // vector: untrusted_endpoint_fragment_before_at_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_fragment_before_at_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_ipv6_literal_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_ipv6_literal_rejected: expected an error"); + } { // vector: untrusted_endpoint_key_with_client_secret_rejected var source = JsonSerializer.Deserialize("null")!; @@ -541,6 +583,76 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea } if (caught is null) throw new SeamConformanceException("untrusted_endpoint_no_key_rejected: expected an error"); } + { + // vector: untrusted_endpoint_percent_encoded_host_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_percent_encoded_host_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_port_mismatch_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_port_mismatch_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_query_before_at_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_query_before_at_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_scheme_mismatch_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_scheme_mismatch_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_suffix_lookalike_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_suffix_lookalike_rejected: expected an error"); + } { // vector: untrusted_endpoint_suffix_spoof_rejected var source = JsonSerializer.Deserialize("null")!; @@ -555,6 +667,48 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea } if (caught is null) throw new SeamConformanceException("untrusted_endpoint_suffix_spoof_rejected: expected an error"); } + { + // vector: untrusted_endpoint_trailing_dot_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_trailing_dot_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_unicode_homoglyph_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_unicode_homoglyph_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_userinfo_lookalike_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_userinfo_lookalike_rejected: expected an error"); + } { // vector: untrusted_endpoint_wildcard_apex_rejected var source = JsonSerializer.Deserialize("null")!; @@ -569,6 +723,20 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea } if (caught is null) throw new SeamConformanceException("untrusted_endpoint_wildcard_apex_rejected: expected an error"); } + { + // vector: untrusted_endpoint_wildcard_boundary_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_wildcard_boundary_rejected: expected an error"); + } } } diff --git a/runtime/go/prompty/model/vector_conformance_test.go b/runtime/go/prompty/model/vector_conformance_test.go index b95316b87..e6c3f22f4 100644 --- a/runtime/go/prompty/model/vector_conformance_test.go +++ b/runtime/go/prompty/model/vector_conformance_test.go @@ -1764,7 +1764,41 @@ func TestVector48LoadConformanceLoadTrustedEndpointBuiltinAllowed(t *testing.T) vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed(t *testing.T) { +func TestVector49LoadConformanceLoadTrustedEndpointDefaultPortAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_default_port_allowed", + "description": "An explicit default port normalizes to the built-in origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-default-port-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector50LoadConformanceLoadTrustedEndpointEnvEndpointAllowed(t *testing.T) { vectorJSON := `{ "name": "trusted_endpoint_env_endpoint_allowed", "description": "An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", @@ -1801,7 +1835,7 @@ func TestVector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed(t *testing vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector50LoadConformanceLoadTrustedEndpointEnvListAllowed(t *testing.T) { +func TestVector51LoadConformanceLoadTrustedEndpointEnvListAllowed(t *testing.T) { vectorJSON := `{ "name": "trusted_endpoint_env_list_allowed", "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", @@ -1838,7 +1872,7 @@ func TestVector50LoadConformanceLoadTrustedEndpointEnvListAllowed(t *testing.T) vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed(t *testing.T) { +func TestVector52LoadConformanceLoadTrustedEndpointLiteralKeyAllowed(t *testing.T) { vectorJSON := `{ "name": "trusted_endpoint_literal_key_allowed", "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", @@ -1873,7 +1907,7 @@ func TestVector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed(t *testing. vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector52LoadConformanceLoadTrustedEndpointOptionAllowed(t *testing.T) { +func TestVector53LoadConformanceLoadTrustedEndpointOptionAllowed(t *testing.T) { vectorJSON := `{ "name": "trusted_endpoint_option_allowed", "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", @@ -1910,7 +1944,7 @@ func TestVector52LoadConformanceLoadTrustedEndpointOptionAllowed(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector53LoadConformanceLoadTrustedEndpointReferenceSkipped(t *testing.T) { +func TestVector54LoadConformanceLoadTrustedEndpointReferenceSkipped(t *testing.T) { vectorJSON := `{ "name": "trusted_endpoint_reference_skipped", "description": "kind: reference connections carry no endpoint and are not subject to the trust check", @@ -1944,7 +1978,41 @@ func TestVector53LoadConformanceLoadTrustedEndpointReferenceSkipped(t *testing.T vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector54LoadConformanceLoadUntrustedEndpointAmbientRejected(t *testing.T) { +func TestVector55LoadConformanceLoadTrustedEndpointUserinfoAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_userinfo_allowed", + "description": "Userinfo is ignored; the destination host api.openai.com is trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-userinfo-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector56LoadConformanceLoadUntrustedEndpointAmbientRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_ambient_rejected", "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)", @@ -1974,7 +2042,7 @@ func TestVector54LoadConformanceLoadUntrustedEndpointAmbientRejected(t *testing. vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector55LoadConformanceLoadUntrustedEndpointBackslashRejected(t *testing.T) { +func TestVector57LoadConformanceLoadUntrustedEndpointBackslashRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_backslash_rejected", "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", @@ -2003,7 +2071,7 @@ func TestVector55LoadConformanceLoadUntrustedEndpointBackslashRejected(t *testin vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testing.T) { +func TestVector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_env_default_rejected", "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", @@ -2033,7 +2101,7 @@ func TestVector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testi vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T) { +func TestVector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_env_key_rejected", "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", @@ -2066,7 +2134,7 @@ func TestVector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *testing.T) { +func TestVector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_file_connection_rejected", "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", @@ -2098,7 +2166,65 @@ func TestVector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected(t *testing.T) { +func TestVector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_fragment_before_at_rejected", + "description": "A '#' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-fragment-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com#@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_ipv6_literal_rejected", + "description": "An IPv6 literal is not a trusted origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ipv6-literal", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://[::1]:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_key_with_client_secret_rejected", "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", @@ -2128,7 +2254,7 @@ func TestVector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected(t *testing.T) { +func TestVector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_mismatched_credential_rejected", "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", @@ -2158,7 +2284,7 @@ func TestVector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejecte vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector61LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) { +func TestVector65LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_no_key_rejected", "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", @@ -2187,7 +2313,152 @@ func TestVector61LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *testing.T) { +func TestVector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_percent_encoded_host_rejected", + "description": "Hosts are compared without percent-decoding, so an encoded host never matches", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-percent-encoded-host", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com%2eevil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_port_mismatch_rejected", + "description": "A non-default port is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-port-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com:8443" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_query_before_at_rejected", + "description": "A '?' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-query-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com?@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_scheme_mismatch_rejected", + "description": "Built-in origins are https; http on the same host is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-scheme-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_suffix_lookalike_rejected", + "description": "A trusted host as a label prefix of another domain is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-suffix-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com.evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_suffix_spoof_rejected", "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", @@ -2219,7 +2490,94 @@ func TestVector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *test vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *testing.T) { +func TestVector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_trailing_dot_rejected", + "description": "Hosts are compared exactly; a trailing-dot host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-trailing-dot", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com." + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_unicode_homoglyph_rejected", + "description": "A non-ASCII lookalike host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-unicode-homoglyph", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://аpi.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_userinfo_lookalike_rejected", + "description": "A trusted host in userinfo is not the destination host", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-userinfo-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com@evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_wildcard_apex_rejected", "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", @@ -2248,7 +2606,36 @@ func TestVector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *tes vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector64MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { +func TestVector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_wildcard_boundary_rejected", + "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-wildcard-boundary", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evilservices.ai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector77MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorJSON := `{ "name": "clear_by_category_and_all", "description": "Clear removes only the requested category, or all memories when no category is supplied.", @@ -2301,7 +2688,7 @@ func TestVector64MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector65MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { +func TestVector78MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { vectorJSON := `{ "name": "format_core_and_recall_results", "description": "Formatting injects only core memories into system prompts and formats recall results with category and tags.", @@ -2342,7 +2729,7 @@ func TestVector65MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector66MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { +func TestVector79MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { vectorJSON := `{ "name": "recall_empty_query_and_limit", "description": "Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.", @@ -2396,7 +2783,7 @@ func TestVector66MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector67MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { +func TestVector80MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { vectorJSON := `{ "name": "recall_ranks_by_weighted_score", "description": "Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.", @@ -2454,7 +2841,7 @@ func TestVector67MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { +func TestVector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { vectorJSON := `{ "name": "recall_score_ties_preserve_insertion_order", "description": "Recall preserves insertion order for memories with identical scores.", @@ -2511,7 +2898,7 @@ func TestVector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector69MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { +func TestVector82MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorJSON := `{ "name": "recall_tags_and_core_boost", "description": "Recall gives tag matches higher weight than content matches and boosts matching core memories.", @@ -2571,7 +2958,7 @@ func TestVector69MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { +func TestVector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { vectorJSON := `{ "name": "recall_tokenization_punctuation_unicode_and_dedup", "description": "Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.", @@ -2631,7 +3018,7 @@ func TestVector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAnd vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector71MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { +func TestVector84MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_by_tags", "description": "Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.", @@ -2694,7 +3081,7 @@ func TestVector71MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { +func TestVector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_missing_and_empty_tags", "description": "Remember treats missing and empty tag lists as equivalent for core-memory deduplication.", @@ -2749,7 +3136,7 @@ func TestVector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector73MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { +func TestVector86MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { vectorJSON := `{ "name": "remember_evicts_archival_first", "description": "Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.", @@ -2809,7 +3196,7 @@ func TestVector73MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing. vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector74MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { +func TestVector87MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorJSON := `{ "name": "snapshot_port_round_trip", "description": "A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.", @@ -2848,7 +3235,7 @@ func TestVector74MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector75MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { +func TestVector88MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { vectorJSON := `{ "name": "update_index_out_of_range_error", "description": "Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.", @@ -2881,7 +3268,7 @@ func TestVector75MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector76TurnConformanceReplayMaxIterations(t *testing.T) { +func TestVector89TurnConformanceReplayMaxIterations(t *testing.T) { vectorJSON := `{ "name": "max_iterations", "stage": "replay", @@ -2917,7 +3304,7 @@ func TestVector76TurnConformanceReplayMaxIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector77TurnConformanceReplayNoTool(t *testing.T) { +func TestVector90TurnConformanceReplayNoTool(t *testing.T) { vectorJSON := `{ "name": "no_tool", "stage": "replay", @@ -2949,7 +3336,7 @@ func TestVector77TurnConformanceReplayNoTool(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector78TurnConformanceReplayPermissionDenied(t *testing.T) { +func TestVector91TurnConformanceReplayPermissionDenied(t *testing.T) { vectorJSON := `{ "name": "permission_denied", "stage": "replay", @@ -2984,7 +3371,7 @@ func TestVector78TurnConformanceReplayPermissionDenied(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector79TurnConformanceReplayToolFailure(t *testing.T) { +func TestVector92TurnConformanceReplayToolFailure(t *testing.T) { vectorJSON := `{ "name": "tool_failure", "stage": "replay", @@ -3021,7 +3408,7 @@ func TestVector79TurnConformanceReplayToolFailure(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector80TurnConformanceReplayToolSuccess(t *testing.T) { +func TestVector93TurnConformanceReplayToolSuccess(t *testing.T) { vectorJSON := `{ "name": "tool_success", "stage": "replay", @@ -3058,7 +3445,7 @@ func TestVector80TurnConformanceReplayToolSuccess(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector81TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { +func TestVector94TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorJSON := `{ "name": "assistant_tool_calls_metadata", "description": "When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.", @@ -3211,7 +3598,7 @@ func TestVector81TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector82TurnConformanceRunAsyncToolFunction(t *testing.T) { +func TestVector95TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorJSON := `{ "name": "async_tool_function", "description": "Agent loop correctly handles async tool functions", @@ -3322,7 +3709,7 @@ func TestVector82TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector83TurnConformanceRunBindingsInjected(t *testing.T) { +func TestVector96TurnConformanceRunBindingsInjected(t *testing.T) { vectorJSON := `{ "name": "bindings_injected", "description": "Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.", @@ -3453,7 +3840,7 @@ func TestVector83TurnConformanceRunBindingsInjected(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector84TurnConformanceRunCancellationBeforeLlm(t *testing.T) { +func TestVector97TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorJSON := `{ "name": "cancellation_before_llm", "description": "§13.2 Cancellation — Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.", @@ -3515,7 +3902,7 @@ func TestVector84TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector85TurnConformanceRunCancellationBetweenIterations(t *testing.T) { +func TestVector98TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_iterations", "description": "§13.2 Cancellation — Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.", @@ -3646,7 +4033,7 @@ func TestVector85TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector86TurnConformanceRunCancellationBetweenTools(t *testing.T) { +func TestVector99TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_tools", "description": "§13.2 Cancellation — LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.", @@ -3786,7 +4173,7 @@ func TestVector86TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector87TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { +func TestVector100TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorJSON := `{ "name": "context_no_trim_when_fits", "description": "§13.3 Context Window — Messages fit within the context budget. No trimming occurs and all messages are preserved.", @@ -3858,7 +4245,7 @@ func TestVector87TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector88TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { +func TestVector101TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { vectorJSON := `{ "name": "context_preserves_system_messages", "description": "§13.3 Context Window — Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.", @@ -3966,7 +4353,7 @@ func TestVector88TurnConformanceRunContextPreservesSystemMessages(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector89TurnConformanceRunContextTrimBasic(t *testing.T) { +func TestVector102TurnConformanceRunContextTrimBasic(t *testing.T) { vectorJSON := `{ "name": "context_trim_basic", "description": "§13.3 Context Window — Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.", @@ -4125,7 +4512,7 @@ func TestVector89TurnConformanceRunContextTrimBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector90TurnConformanceRunEmptyToolResult(t *testing.T) { +func TestVector103TurnConformanceRunEmptyToolResult(t *testing.T) { vectorJSON := `{ "name": "empty_tool_result", "description": "Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content — it must not be skipped or filtered out.", @@ -4241,7 +4628,7 @@ func TestVector90TurnConformanceRunEmptyToolResult(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector91TurnConformanceRunEventsBasicToolLoop(t *testing.T) { +func TestVector104TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorJSON := `{ "name": "events_basic_tool_loop", "description": "§13.1 Events — A 2-turn tool-call loop emits the full event lifecycle: status → tool_call_start → tool_result → messages_updated → done.", @@ -4388,7 +4775,7 @@ func TestVector91TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector92TurnConformanceRunEventsErrorLogged(t *testing.T) { +func TestVector105TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorJSON := `{ "name": "events_error_logged", "description": "§13.1 Events — A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.", @@ -4515,7 +4902,7 @@ func TestVector92TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector93TurnConformanceRunEventsNoTools(t *testing.T) { +func TestVector106TurnConformanceRunEventsNoTools(t *testing.T) { vectorJSON := `{ "name": "events_no_tools", "description": "§13.1 Events — Single-turn completion with no tool calls. Only the done event is emitted.", @@ -4594,7 +4981,7 @@ func TestVector93TurnConformanceRunEventsNoTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector94TurnConformanceRunGuardrailAllPass(t *testing.T) { +func TestVector107TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorJSON := `{ "name": "guardrail_all_pass", "description": "§13.4 Guardrails — All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.", @@ -4718,7 +5105,7 @@ func TestVector94TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector95TurnConformanceRunGuardrailInputDeny(t *testing.T) { +func TestVector108TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_input_deny", "description": "§13.4 Guardrails — Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.", @@ -4775,7 +5162,7 @@ func TestVector95TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector96TurnConformanceRunGuardrailOutputDeny(t *testing.T) { +func TestVector109TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_output_deny", "description": "§13.4 Guardrails — Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.", @@ -4854,7 +5241,7 @@ func TestVector96TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector97TurnConformanceRunGuardrailToolDeny(t *testing.T) { +func TestVector110TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_tool_deny", "description": "§13.4 Guardrails — LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.", @@ -5009,7 +5396,7 @@ func TestVector97TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector98TurnConformanceRunMaxIterationsExceeded(t *testing.T) { +func TestVector111TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorJSON := `{ "name": "max_iterations_exceeded", "description": "LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.", @@ -5520,7 +5907,7 @@ func TestVector98TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector99TurnConformanceRunMultiTurnToolCalls(t *testing.T) { +func TestVector112TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorJSON := `{ "name": "multi_turn_tool_calls", "description": "Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.", @@ -5698,7 +6085,7 @@ func TestVector99TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector100TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { +func TestVector113TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorJSON := `{ "name": "multiple_tool_calls_single_turn", "description": "Turn 1: LLM returns two parallel tool calls — get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.", @@ -5880,7 +6267,7 @@ func TestVector100TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector101TurnConformanceRunNoToolCalls(t *testing.T) { +func TestVector114TurnConformanceRunNoToolCalls(t *testing.T) { vectorJSON := `{ "name": "no_tool_calls", "description": "LLM returns content immediately with no tool calls — the agent loop completes in a single iteration.", @@ -5949,7 +6336,7 @@ func TestVector101TurnConformanceRunNoToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector102TurnConformanceRunParallelToolsBasic(t *testing.T) { +func TestVector115TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_basic", "description": "§13.6 Parallel Tools — LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.", @@ -6123,7 +6510,7 @@ func TestVector102TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector103TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { +func TestVector116TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_guardrail_deny", "description": "§13.6 Parallel Tools + §13.4 Guardrails — 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.", @@ -6315,7 +6702,7 @@ func TestVector103TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector104TurnConformanceRunParallelToolsWithToolError(t *testing.T) { +func TestVector117TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_tool_error", "description": "§13.6 Parallel Tools + tool error — 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.", @@ -6489,7 +6876,7 @@ func TestVector104TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector105TurnConformanceRunSingleToolCall(t *testing.T) { +func TestVector118TurnConformanceRunSingleToolCall(t *testing.T) { vectorJSON := `{ "name": "single_tool_call", "description": "Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.", @@ -6637,7 +7024,7 @@ func TestVector105TurnConformanceRunSingleToolCall(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector106TurnConformanceRunSteeringInjectMessage(t *testing.T) { +func TestVector119TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorJSON := `{ "name": "steering_inject_message", "description": "§13.5 Steering — After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.", @@ -6846,7 +7233,7 @@ func TestVector106TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector107TurnConformanceRunSteeringMultipleMessages(t *testing.T) { +func TestVector120TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorJSON := `{ "name": "steering_multiple_messages", "description": "§13.5 Steering — Two steering messages are queued and both injected before iteration 2. Tests atomic drain — both appear in order.", @@ -7017,7 +7404,7 @@ func TestVector107TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector108TurnConformanceRunToolNotRegisteredError(t *testing.T) { +func TestVector121TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorJSON := `{ "name": "tool_not_registered_error", "description": "LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.", @@ -7102,7 +7489,7 @@ func TestVector108TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector109TurnConformanceRunToolResultMessageFormat(t *testing.T) { +func TestVector122TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorJSON := `{ "name": "tool_result_message_format", "description": "After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.", @@ -7224,7 +7611,7 @@ func TestVector109TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector110TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { +func TestVector123TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorJSON := `{ "name": "cancel_before_context", "stage": "turn", @@ -7257,7 +7644,7 @@ func TestVector110TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector111TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { +func TestVector124TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorJSON := `{ "name": "delegated_provider_state", "stage": "turn", @@ -7325,7 +7712,7 @@ func TestVector111TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector112TurnConformanceRunTurnFinalOutput(t *testing.T) { +func TestVector125TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorJSON := `{ "name": "final_output", "stage": "turn", @@ -7371,7 +7758,7 @@ func TestVector112TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector113TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { +func TestVector126TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "memory_context_is_model_visible", "stage": "turn", @@ -7434,7 +7821,7 @@ func TestVector113TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector114TurnConformanceRunTurnOrderedToolRound(t *testing.T) { +func TestVector127TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorJSON := `{ "name": "ordered_tool_round", "stage": "turn", @@ -7526,7 +7913,7 @@ func TestVector114TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector115TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { +func TestVector128TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "permission_denial_is_model_visible", "stage": "turn", @@ -7578,7 +7965,7 @@ func TestVector115TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testin vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector116WireConformanceToRequestAnthropicImageFormat(t *testing.T) { +func TestVector129WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorJSON := `{ "name": "anthropic_image_format", "description": "§7.5 — Anthropic: ImagePart uses base64 source block with media_type, not image_url format.", @@ -7649,7 +8036,7 @@ func TestVector116WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector117WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { +func TestVector130WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { vectorJSON := `{ "name": "anthropic_max_tokens_required", "description": "§7.5 — Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).", @@ -7705,7 +8092,7 @@ func TestVector117WireConformanceToRequestAnthropicMaxTokensRequired(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector118WireConformanceToRequestAnthropicOptions(t *testing.T) { +func TestVector131WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorJSON := `{ "name": "anthropic_options", "description": "Anthropic: option names map correctly (topK, stopSequences)", @@ -7775,7 +8162,7 @@ func TestVector118WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector119WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { +func TestVector132WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { vectorJSON := `{ "name": "anthropic_system_separate", "description": "§7.5 — Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.", @@ -7843,7 +8230,7 @@ func TestVector119WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector120WireConformanceToRequestAnthropicToolWire(t *testing.T) { +func TestVector133WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorJSON := `{ "name": "anthropic_tool_wire", "description": "Anthropic: tools use input_schema (not parameters), name at top level", @@ -7929,7 +8316,7 @@ func TestVector120WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector121WireConformanceToRequestChatAudioMp3(t *testing.T) { +func TestVector134WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorJSON := `{ "name": "chat_audio_mp3", "description": "§7.1.2 — AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.", @@ -7988,7 +8375,7 @@ func TestVector121WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector122WireConformanceToRequestChatAudioPart(t *testing.T) { +func TestVector135WireConformanceToRequestChatAudioPart(t *testing.T) { vectorJSON := `{ "name": "chat_audio_part", "description": "§7.1.2 — AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.", @@ -8047,7 +8434,7 @@ func TestVector122WireConformanceToRequestChatAudioPart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector123WireConformanceToRequestChatImageBase64(t *testing.T) { +func TestVector136WireConformanceToRequestChatImageBase64(t *testing.T) { vectorJSON := `{ "name": "chat_image_base64", "description": "Base64 data URI image part preserved as-is in image_url", @@ -8112,7 +8499,7 @@ func TestVector123WireConformanceToRequestChatImageBase64(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector124WireConformanceToRequestChatImagePart(t *testing.T) { +func TestVector137WireConformanceToRequestChatImagePart(t *testing.T) { vectorJSON := `{ "name": "chat_image_part", "description": "Image content part converted to OpenAI image_url format", @@ -8177,7 +8564,7 @@ func TestVector124WireConformanceToRequestChatImagePart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector125WireConformanceToRequestChatMultipartContent(t *testing.T) { +func TestVector138WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorJSON := `{ "name": "chat_multipart_content", "description": "§7.1.1/§7.1.2 — Message with TextPart + ImagePart MUST produce an array of typed content blocks.", @@ -8242,7 +8629,7 @@ func TestVector125WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector126WireConformanceToRequestChatSimple(t *testing.T) { +func TestVector139WireConformanceToRequestChatSimple(t *testing.T) { vectorJSON := `{ "name": "chat_simple", "description": "§7.1.7 — Two messages (system + user), no tools, no options. Minimal chat request.", @@ -8305,7 +8692,7 @@ func TestVector126WireConformanceToRequestChatSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector127WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { +func TestVector140WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { vectorJSON := `{ "name": "chat_single_text_optimized", "description": "§7.1.1 — Message with exactly 1 TextPart MUST produce a plain string content, not an array.", @@ -8355,7 +8742,7 @@ func TestVector127WireConformanceToRequestChatSingleTextOptimized(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector128WireConformanceToRequestChatWithOptions(t *testing.T) { +func TestVector141WireConformanceToRequestChatWithOptions(t *testing.T) { vectorJSON := `{ "name": "chat_with_options", "description": "§7.1.5 — All standard ModelOptions mapped to OpenAI parameter names.", @@ -8424,7 +8811,7 @@ func TestVector128WireConformanceToRequestChatWithOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector129WireConformanceToRequestEmbeddingWire(t *testing.T) { +func TestVector142WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorJSON := `{ "name": "embedding_wire", "description": "§7.2 — Embedding request: text extracted from messages, single string input (not array).", @@ -8469,7 +8856,7 @@ func TestVector129WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector130WireConformanceToRequestImageWire(t *testing.T) { +func TestVector143WireConformanceToRequestImageWire(t *testing.T) { vectorJSON := `{ "name": "image_wire", "description": "§7.3 — Image generation request: prompt extracted from last user message.", @@ -8514,7 +8901,7 @@ func TestVector130WireConformanceToRequestImageWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector131WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { +func TestVector144WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorJSON := `{ "name": "kind_to_json_type_mapping", "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.", @@ -8634,7 +9021,7 @@ func TestVector131WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector132WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { +func TestVector145WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { vectorJSON := `{ "name": "options_additional_properties", "description": "§7.1.5 — additionalProperties from ModelOptions MUST be merged into the request as top-level keys.", @@ -8689,7 +9076,7 @@ func TestVector132WireConformanceToRequestOptionsAdditionalProperties(t *testing vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector133WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { +func TestVector146WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { vectorJSON := `{ "name": "options_max_completion_tokens", "description": "§7.1.5 — maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.", @@ -8742,7 +9129,7 @@ func TestVector133WireConformanceToRequestOptionsMaxCompletionTokens(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector134WireConformanceToRequestOptionsStopSequences(t *testing.T) { +func TestVector147WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorJSON := `{ "name": "options_stop_sequences", "description": "§7.1.5 — stopSequences MUST map to 'stop' in the OpenAI wire format.", @@ -8801,7 +9188,7 @@ func TestVector134WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector135WireConformanceToRequestResponsesSimple(t *testing.T) { +func TestVector148WireConformanceToRequestResponsesSimple(t *testing.T) { vectorJSON := `{ "name": "responses_simple", "description": "Responses API: system becomes instructions, messages become input items", @@ -8861,7 +9248,7 @@ func TestVector135WireConformanceToRequestResponsesSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector136WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { +func TestVector149WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "responses_structured_output", "description": "Responses API: structured output uses text.format.json_schema", @@ -8949,7 +9336,7 @@ func TestVector136WireConformanceToRequestResponsesStructuredOutput(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector137WireConformanceToRequestResponsesWithTools(t *testing.T) { +func TestVector150WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorJSON := `{ "name": "responses_with_tools", "description": "Responses API: tools use flat format (no nested function key)", @@ -9030,7 +9417,7 @@ func TestVector137WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector138WireConformanceToRequestStructuredOutput(t *testing.T) { +func TestVector151WireConformanceToRequestStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "structured_output", "description": "§7.1.6 — outputs converted to response_format with json_schema, strict=true, additionalProperties=false.", @@ -9114,7 +9501,7 @@ func TestVector138WireConformanceToRequestStructuredOutput(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector139WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { +func TestVector152WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { vectorJSON := `{ "name": "structured_output_nested_optional", "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", @@ -9217,7 +9604,7 @@ func TestVector139WireConformanceToRequestStructuredOutputNestedOptional(t *test vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector140WireConformanceToRequestToolsBindingsStripped(t *testing.T) { +func TestVector153WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorJSON := `{ "name": "tools_bindings_stripped", "description": "§7.1.3 — Parameters listed in bindings MUST be stripped from wire tools (properties AND required).", @@ -9310,7 +9697,7 @@ func TestVector140WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector141WireConformanceToRequestToolsFunctionWire(t *testing.T) { +func TestVector154WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorJSON := `{ "name": "tools_function_wire", "description": "§7.1.3 — FunctionTool projected as OpenAI function definition with JSON Schema parameters.", @@ -9393,7 +9780,7 @@ func TestVector141WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector142WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { +func TestVector155WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorJSON := `{ "name": "tools_null_when_empty", "description": "§7.1.3 — When no tools are defined, the tools key MUST be absent from the request entirely.", @@ -9443,7 +9830,7 @@ func TestVector142WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector143WireConformanceToRequestToolsStrictMode(t *testing.T) { +func TestVector156WireConformanceToRequestToolsStrictMode(t *testing.T) { vectorJSON := `{ "name": "tools_strict_mode", "description": "§7.1.3 — FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.", diff --git a/runtime/go/prompty/model/vectorconformance/vector_conformance.go b/runtime/go/prompty/model/vectorconformance/vector_conformance.go index 0e8fcea69..e50d1b317 100644 --- a/runtime/go/prompty/model/vectorconformance/vector_conformance.go +++ b/runtime/go/prompty/model/vectorconformance/vector_conformance.go @@ -1113,6 +1113,32 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("trusted_endpoint_builtin_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) } } + // vector: trusted_endpoint_default_port_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_default_port_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_default_port_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_default_port_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_default_port_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_default_port_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_default_port_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } // vector: trusted_endpoint_env_endpoint_allowed { var source interface{} @@ -1243,6 +1269,32 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("trusted_endpoint_reference_skipped misrouted: got %#v, want %#v", actualValue, expectedValue) } } + // vector: trusted_endpoint_userinfo_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_userinfo_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_userinfo_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_userinfo_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_userinfo_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_userinfo_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_userinfo_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } // vector: untrusted_endpoint_ambient_rejected { var source interface{} @@ -1298,6 +1350,28 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("untrusted_endpoint_file_connection_rejected: expected an error, got nil") } } + // vector: untrusted_endpoint_fragment_before_at_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_fragment_before_at_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_fragment_before_at_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_ipv6_literal_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_ipv6_literal_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_ipv6_literal_rejected: expected an error, got nil") + } + } // vector: untrusted_endpoint_key_with_client_secret_rejected { var source interface{} @@ -1331,6 +1405,61 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("untrusted_endpoint_no_key_rejected: expected an error, got nil") } } + // vector: untrusted_endpoint_percent_encoded_host_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_percent_encoded_host_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_percent_encoded_host_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_port_mismatch_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_port_mismatch_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_port_mismatch_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_query_before_at_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_query_before_at_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_query_before_at_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_scheme_mismatch_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_scheme_mismatch_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_scheme_mismatch_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_suffix_lookalike_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_suffix_lookalike_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_suffix_lookalike_rejected: expected an error, got nil") + } + } // vector: untrusted_endpoint_suffix_spoof_rejected { var source interface{} @@ -1342,6 +1471,39 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("untrusted_endpoint_suffix_spoof_rejected: expected an error, got nil") } } + // vector: untrusted_endpoint_trailing_dot_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_trailing_dot_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_trailing_dot_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_unicode_homoglyph_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_unicode_homoglyph_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_unicode_homoglyph_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_userinfo_lookalike_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_userinfo_lookalike_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_userinfo_lookalike_rejected: expected an error, got nil") + } + } // vector: untrusted_endpoint_wildcard_apex_rejected { var source interface{} @@ -1353,4 +1515,15 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("untrusted_endpoint_wildcard_apex_rejected: expected an error, got nil") } } + // vector: untrusted_endpoint_wildcard_boundary_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_wildcard_boundary_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_wildcard_boundary_rejected: expected an error, got nil") + } + } } diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java index 06edf857f..66c82ffec 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java @@ -613,6 +613,19 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex if (!actual.toJson().equals(TypraJson.stringify(expected))) throw new AssertionError("trusted_endpoint_builtin_allowed misrouted"); } + { + // vector: trusted_endpoint_default_port_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-default-port-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_default_port_allowed misrouted"); + } { // vector: trusted_endpoint_env_endpoint_allowed StringBuilder inputJson = new StringBuilder(); @@ -678,6 +691,19 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex if (!actual.toJson().equals(TypraJson.stringify(expected))) throw new AssertionError("trusted_endpoint_reference_skipped misrouted"); } + { + // vector: trusted_endpoint_userinfo_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-userinfo-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_userinfo_allowed misrouted"); + } { // vector: untrusted_endpoint_ambient_rejected StringBuilder inputJson = new StringBuilder(); @@ -748,6 +774,34 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex } if (caught == null) throw new AssertionError("untrusted_endpoint_file_connection_rejected: expected an error"); } + { + // vector: untrusted_endpoint_fragment_before_at_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_fragment_before_at_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_ipv6_literal_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_ipv6_literal_rejected: expected an error"); + } { // vector: untrusted_endpoint_key_with_client_secret_rejected StringBuilder inputJson = new StringBuilder(); @@ -790,6 +844,76 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex } if (caught == null) throw new AssertionError("untrusted_endpoint_no_key_rejected: expected an error"); } + { + // vector: untrusted_endpoint_percent_encoded_host_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_percent_encoded_host_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_port_mismatch_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_port_mismatch_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_query_before_at_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_query_before_at_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_scheme_mismatch_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_scheme_mismatch_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_suffix_lookalike_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_suffix_lookalike_rejected: expected an error"); + } { // vector: untrusted_endpoint_suffix_spoof_rejected StringBuilder inputJson = new StringBuilder(); @@ -804,6 +928,48 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex } if (caught == null) throw new AssertionError("untrusted_endpoint_suffix_spoof_rejected: expected an error"); } + { + // vector: untrusted_endpoint_trailing_dot_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_trailing_dot_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_unicode_homoglyph_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\\u0430pi.openai.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_unicode_homoglyph_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_userinfo_lookalike_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_userinfo_lookalike_rejected: expected an error"); + } { // vector: untrusted_endpoint_wildcard_apex_rejected StringBuilder inputJson = new StringBuilder(); @@ -818,6 +984,20 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex } if (caught == null) throw new AssertionError("untrusted_endpoint_wildcard_apex_rejected: expected an error"); } + { + // vector: untrusted_endpoint_wildcard_boundary_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_wildcard_boundary_rejected: expected an error"); + } } } diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java index ac2d8dcc0..a1c756148 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java @@ -400,665 +400,756 @@ private static void vector48LoadConformanceLoadTrustedEndpointBuiltinAllowed() t VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() throws Exception { + private static void vector49LoadConformanceLoadTrustedEndpointDefaultPortAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_default_port_allowed\",\"description\":\"An explicit default port normalizes to the built-in origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-default-port-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector50LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"trusted_endpoint_env_endpoint_allowed\",\"description\":\"An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector50LoadConformanceLoadTrustedEndpointEnvListAllowed() throws Exception { + private static void vector51LoadConformanceLoadTrustedEndpointEnvListAllowed() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"trusted_endpoint_env_list_allowed\",\"description\":\"PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() throws Exception { + private static void vector52LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"trusted_endpoint_literal_key_allowed\",\"description\":\"A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector52LoadConformanceLoadTrustedEndpointOptionAllowed() throws Exception { + private static void vector53LoadConformanceLoadTrustedEndpointOptionAllowed() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"trusted_endpoint_option_allowed\",\"description\":\"A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector53LoadConformanceLoadTrustedEndpointReferenceSkipped() throws Exception { + private static void vector54LoadConformanceLoadTrustedEndpointReferenceSkipped() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"trusted_endpoint_reference_skipped\",\"description\":\"kind: reference connections carry no endpoint and are not subject to the trust check\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector54LoadConformanceLoadUntrustedEndpointAmbientRejected() throws Exception { + private static void vector55LoadConformanceLoadTrustedEndpointUserinfoAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_userinfo_allowed\",\"description\":\"Userinfo is ignored; the destination host api.openai.com is trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-userinfo-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector56LoadConformanceLoadUntrustedEndpointAmbientRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_ambient_rejected\",\"description\":\"A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec \\u00a72.5.1)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector55LoadConformanceLoadUntrustedEndpointBackslashRejected() throws Exception { + private static void vector57LoadConformanceLoadUntrustedEndpointBackslashRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_backslash_rejected\",\"description\":\"A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() throws Exception { + private static void vector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected() throws Exception { + private static void vector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected() throws Exception { + private static void vector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() throws Exception { + private static void vector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_fragment_before_at_rejected\",\"description\":\"A '#' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_ipv6_literal_rejected\",\"description\":\"An IPv6 literal is not a trusted origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() throws Exception { + private static void vector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector61LoadConformanceLoadUntrustedEndpointNoKeyRejected() throws Exception { + private static void vector65LoadConformanceLoadUntrustedEndpointNoKeyRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() throws Exception { + private static void vector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_percent_encoded_host_rejected\",\"description\":\"Hosts are compared without percent-decoding, so an encoded host never matches\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_port_mismatch_rejected\",\"description\":\"A non-default port is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_query_before_at_rejected\",\"description\":\"A '?' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_scheme_mismatch_rejected\",\"description\":\"Built-in origins are https; http on the same host is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_suffix_lookalike_rejected\",\"description\":\"A trusted host as a label prefix of another domain is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected() throws Exception { + private static void vector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_trailing_dot_rejected\",\"description\":\"Hosts are compared exactly; a trailing-dot host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_unicode_homoglyph_rejected\",\"description\":\"A non-ASCII lookalike host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\\u0430pi.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_userinfo_lookalike_rejected\",\"description\":\"A trusted host in userinfo is not the destination host\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector64MemoryConformanceOperateClearByCategoryAndAll() throws Exception { + private static void vector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_wildcard_boundary_rejected\",\"description\":\"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector77MemoryConformanceOperateClearByCategoryAndAll() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector65MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { + private static void vector78MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector66MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { + private static void vector79MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector67MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { + private static void vector80MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { + private static void vector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector69MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { + private static void vector82MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { + private static void vector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\\u00e9\"]},{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\\u00e9\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector71MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { + private static void vector84MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { + private static void vector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector73MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { + private static void vector86MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector74MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { + private static void vector87MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector75MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { + private static void vector88MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector76TurnConformanceReplayMaxIterations() throws Exception { + private static void vector89TurnConformanceReplayMaxIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector77TurnConformanceReplayNoTool() throws Exception { + private static void vector90TurnConformanceReplayNoTool() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector78TurnConformanceReplayPermissionDenied() throws Exception { + private static void vector91TurnConformanceReplayPermissionDenied() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector79TurnConformanceReplayToolFailure() throws Exception { + private static void vector92TurnConformanceReplayToolFailure() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector80TurnConformanceReplayToolSuccess() throws Exception { + private static void vector93TurnConformanceReplayToolSuccess() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector81TurnConformanceRunAssistantToolCallsMetadata() throws Exception { + private static void vector94TurnConformanceRunAssistantToolCallsMetadata() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector82TurnConformanceRunAsyncToolFunction() throws Exception { + private static void vector95TurnConformanceRunAsyncToolFunction() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector83TurnConformanceRunBindingsInjected() throws Exception { + private static void vector96TurnConformanceRunBindingsInjected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\\u00b0C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector84TurnConformanceRunCancellationBeforeLlm() throws Exception { + private static void vector97TurnConformanceRunCancellationBeforeLlm() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_before_llm\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector85TurnConformanceRunCancellationBetweenIterations() throws Exception { + private static void vector98TurnConformanceRunCancellationBetweenIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_iterations\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector86TurnConformanceRunCancellationBetweenTools() throws Exception { + private static void vector99TurnConformanceRunCancellationBetweenTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_tools\",\"description\":\"\\u00a713.2 Cancellation \\u2014 LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector87TurnConformanceRunContextNoTrimWhenFits() throws Exception { + private static void vector100TurnConformanceRunContextNoTrimWhenFits() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector88TurnConformanceRunContextPreservesSystemMessages() throws Exception { + private static void vector101TurnConformanceRunContextPreservesSystemMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_preserves_system_messages\",\"description\":\"\\u00a713.3 Context Window \\u2014 Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \\u2014 they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector89TurnConformanceRunContextTrimBasic() throws Exception { + private static void vector102TurnConformanceRunContextTrimBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_trim_basic\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector90TurnConformanceRunEmptyToolResult() throws Exception { + private static void vector103TurnConformanceRunEmptyToolResult() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \\u2014 it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector91TurnConformanceRunEventsBasicToolLoop() throws Exception { + private static void vector104TurnConformanceRunEventsBasicToolLoop() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_basic_tool_loop\",\"description\":\"\\u00a713.1 Events \\u2014 A 2-turn tool-call loop emits the full event lifecycle: status \\u2192 tool_call_start \\u2192 tool_result \\u2192 messages_updated \\u2192 done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\\u00b0F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector92TurnConformanceRunEventsErrorLogged() throws Exception { + private static void vector105TurnConformanceRunEventsErrorLogged() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_error_logged\",\"description\":\"\\u00a713.1 Events \\u0097 A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \\u0097 this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector93TurnConformanceRunEventsNoTools() throws Exception { + private static void vector106TurnConformanceRunEventsNoTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_no_tools\",\"description\":\"\\u00a713.1 Events \\u2014 Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector94TurnConformanceRunGuardrailAllPass() throws Exception { + private static void vector107TurnConformanceRunGuardrailAllPass() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_all_pass\",\"description\":\"\\u00a713.4 Guardrails \\u2014 All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector95TurnConformanceRunGuardrailInputDeny() throws Exception { + private static void vector108TurnConformanceRunGuardrailInputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_input_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector96TurnConformanceRunGuardrailOutputDeny() throws Exception { + private static void vector109TurnConformanceRunGuardrailOutputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_output_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector97TurnConformanceRunGuardrailToolDeny() throws Exception { + private static void vector110TurnConformanceRunGuardrailToolDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_tool_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \\u2014 its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector98TurnConformanceRunMaxIterationsExceeded() throws Exception { + private static void vector111TurnConformanceRunMaxIterationsExceeded() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\\u00b0F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\\u00b0F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\\u00b0F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\\u00b0F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\\u00b0F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\\u00b0F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\\u00b0F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\\u00b0F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\\u00b0F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector99TurnConformanceRunMultiTurnToolCalls() throws Exception { + private static void vector112TurnConformanceRunMultiTurnToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\\u00b0C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector100TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { + private static void vector113TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \\u2014 get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\\u00b0F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector101TurnConformanceRunNoToolCalls() throws Exception { + private static void vector114TurnConformanceRunNoToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \\u2014 the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector102TurnConformanceRunParallelToolsBasic() throws Exception { + private static void vector115TurnConformanceRunParallelToolsBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_basic\",\"description\":\"\\u00a713.6 Parallel Tools \\u2014 LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector103TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { + private static void vector116TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\\u00a713.6 Parallel Tools + \\u00a713.4 Guardrails \\u2014 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector104TurnConformanceRunParallelToolsWithToolError() throws Exception { + private static void vector117TurnConformanceRunParallelToolsWithToolError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\\u00a713.6 Parallel Tools + tool error \\u2014 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector105TurnConformanceRunSingleToolCall() throws Exception { + private static void vector118TurnConformanceRunSingleToolCall() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector106TurnConformanceRunSteeringInjectMessage() throws Exception { + private static void vector119TurnConformanceRunSteeringInjectMessage() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_inject_message\",\"description\":\"\\u00a713.5 Steering \\u2014 After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector107TurnConformanceRunSteeringMultipleMessages() throws Exception { + private static void vector120TurnConformanceRunSteeringMultipleMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_multiple_messages\",\"description\":\"\\u00a713.5 Steering \\u2014 Two steering messages are queued and both injected before iteration 2. Tests atomic drain \\u2014 both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\\u00b0F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector108TurnConformanceRunToolNotRegisteredError() throws Exception { + private static void vector121TurnConformanceRunToolNotRegisteredError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector109TurnConformanceRunToolResultMessageFormat() throws Exception { + private static void vector122TurnConformanceRunToolResultMessageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\\u00b0F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\\u00b0F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector110TurnConformanceRunTurnCancelBeforeContext() throws Exception { + private static void vector123TurnConformanceRunTurnCancelBeforeContext() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector111TurnConformanceRunTurnDelegatedProviderState() throws Exception { + private static void vector124TurnConformanceRunTurnDelegatedProviderState() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector112TurnConformanceRunTurnFinalOutput() throws Exception { + private static void vector125TurnConformanceRunTurnFinalOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector113TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { + private static void vector126TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector114TurnConformanceRunTurnOrderedToolRound() throws Exception { + private static void vector127TurnConformanceRunTurnOrderedToolRound() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector115TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { + private static void vector128TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector116WireConformanceToRequestAnthropicImageFormat() throws Exception { + private static void vector129WireConformanceToRequestAnthropicImageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_image_format\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector117WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { + private static void vector130WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector118WireConformanceToRequestAnthropicOptions() throws Exception { + private static void vector131WireConformanceToRequestAnthropicOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector119WireConformanceToRequestAnthropicSystemSeparate() throws Exception { + private static void vector132WireConformanceToRequestAnthropicSystemSeparate() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_system_separate\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector120WireConformanceToRequestAnthropicToolWire() throws Exception { + private static void vector133WireConformanceToRequestAnthropicToolWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector121WireConformanceToRequestChatAudioMp3() throws Exception { + private static void vector134WireConformanceToRequestChatAudioMp3() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_mp3\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector122WireConformanceToRequestChatAudioPart() throws Exception { + private static void vector135WireConformanceToRequestChatAudioPart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_part\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector123WireConformanceToRequestChatImageBase64() throws Exception { + private static void vector136WireConformanceToRequestChatImageBase64() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector124WireConformanceToRequestChatImagePart() throws Exception { + private static void vector137WireConformanceToRequestChatImagePart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector125WireConformanceToRequestChatMultipartContent() throws Exception { + private static void vector138WireConformanceToRequestChatMultipartContent() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_multipart_content\",\"description\":\"\\u00a77.1.1/\\u00a77.1.2 \\u2014 Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector126WireConformanceToRequestChatSimple() throws Exception { + private static void vector139WireConformanceToRequestChatSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_simple\",\"description\":\"\\u00a77.1.7 \\u2014 Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector127WireConformanceToRequestChatSingleTextOptimized() throws Exception { + private static void vector140WireConformanceToRequestChatSingleTextOptimized() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_single_text_optimized\",\"description\":\"\\u00a77.1.1 \\u2014 Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector128WireConformanceToRequestChatWithOptions() throws Exception { + private static void vector141WireConformanceToRequestChatWithOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_with_options\",\"description\":\"\\u00a77.1.5 \\u2014 All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector129WireConformanceToRequestEmbeddingWire() throws Exception { + private static void vector142WireConformanceToRequestEmbeddingWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"embedding_wire\",\"description\":\"\\u00a77.2 \\u2014 Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector130WireConformanceToRequestImageWire() throws Exception { + private static void vector143WireConformanceToRequestImageWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"image_wire\",\"description\":\"\\u00a77.3 \\u2014 Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector131WireConformanceToRequestKindToJsonTypeMapping() throws Exception { + private static void vector144WireConformanceToRequestKindToJsonTypeMapping() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\\u00a77.1.4 \\u2014 All Property kind values mapped to JSON Schema types: string\\u2192string, integer\\u2192integer, float\\u2192number, boolean\\u2192boolean, array\\u2192array, object\\u2192object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector132WireConformanceToRequestOptionsAdditionalProperties() throws Exception { + private static void vector145WireConformanceToRequestOptionsAdditionalProperties() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_additional_properties\",\"description\":\"\\u00a77.1.5 \\u2014 additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector133WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { + private static void vector146WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_max_completion_tokens\",\"description\":\"\\u00a77.1.5 \\u2014 maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector134WireConformanceToRequestOptionsStopSequences() throws Exception { + private static void vector147WireConformanceToRequestOptionsStopSequences() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_stop_sequences\",\"description\":\"\\u00a77.1.5 \\u2014 stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector135WireConformanceToRequestResponsesSimple() throws Exception { + private static void vector148WireConformanceToRequestResponsesSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector136WireConformanceToRequestResponsesStructuredOutput() throws Exception { + private static void vector149WireConformanceToRequestResponsesStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector137WireConformanceToRequestResponsesWithTools() throws Exception { + private static void vector150WireConformanceToRequestResponsesWithTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector138WireConformanceToRequestStructuredOutput() throws Exception { + private static void vector151WireConformanceToRequestStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output\",\"description\":\"\\u00a77.1.6 \\u2014 outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector139WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { + private static void vector152WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output_nested_optional\",\"description\":\"\\u00a77.1.4/\\u00a77.1.6 \\u2014 OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector140WireConformanceToRequestToolsBindingsStripped() throws Exception { + private static void vector153WireConformanceToRequestToolsBindingsStripped() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_bindings_stripped\",\"description\":\"\\u00a77.1.3 \\u2014 Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector141WireConformanceToRequestToolsFunctionWire() throws Exception { + private static void vector154WireConformanceToRequestToolsFunctionWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_function_wire\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector142WireConformanceToRequestToolsNullWhenEmpty() throws Exception { + private static void vector155WireConformanceToRequestToolsNullWhenEmpty() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_null_when_empty\",\"description\":\"\\u00a77.1.3 \\u2014 When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector143WireConformanceToRequestToolsStrictMode() throws Exception { + private static void vector156WireConformanceToRequestToolsStrictMode() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_strict_mode\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); @@ -1116,101 +1207,114 @@ public static void run() { failed += runCase("LoadConformance.load:tools_mcp_load", VectorConformanceTests::vector46LoadConformanceLoadToolsMcpLoad); failed += runCase("LoadConformance.load:tools_openapi_load", VectorConformanceTests::vector47LoadConformanceLoadToolsOpenapiLoad); failed += runCase("LoadConformance.load:trusted_endpoint_builtin_allowed", VectorConformanceTests::vector48LoadConformanceLoadTrustedEndpointBuiltinAllowed); - failed += runCase("LoadConformance.load:trusted_endpoint_env_endpoint_allowed", VectorConformanceTests::vector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed); - failed += runCase("LoadConformance.load:trusted_endpoint_env_list_allowed", VectorConformanceTests::vector50LoadConformanceLoadTrustedEndpointEnvListAllowed); - failed += runCase("LoadConformance.load:trusted_endpoint_literal_key_allowed", VectorConformanceTests::vector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed); - failed += runCase("LoadConformance.load:trusted_endpoint_option_allowed", VectorConformanceTests::vector52LoadConformanceLoadTrustedEndpointOptionAllowed); - failed += runCase("LoadConformance.load:trusted_endpoint_reference_skipped", VectorConformanceTests::vector53LoadConformanceLoadTrustedEndpointReferenceSkipped); - failed += runCase("LoadConformance.load:untrusted_endpoint_ambient_rejected", VectorConformanceTests::vector54LoadConformanceLoadUntrustedEndpointAmbientRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_backslash_rejected", VectorConformanceTests::vector55LoadConformanceLoadUntrustedEndpointBackslashRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_env_default_rejected", VectorConformanceTests::vector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_env_key_rejected", VectorConformanceTests::vector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_file_connection_rejected", VectorConformanceTests::vector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", VectorConformanceTests::vector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", VectorConformanceTests::vector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_no_key_rejected", VectorConformanceTests::vector61LoadConformanceLoadUntrustedEndpointNoKeyRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", VectorConformanceTests::vector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", VectorConformanceTests::vector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected); - failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector64MemoryConformanceOperateClearByCategoryAndAll); - failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector65MemoryConformanceOperateFormatCoreAndRecallResults); - failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector66MemoryConformanceOperateRecallEmptyQueryAndLimit); - failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector67MemoryConformanceOperateRecallRanksByWeightedScore); - failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); - failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector69MemoryConformanceOperateRecallTagsAndCoreBoost); - failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); - failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector71MemoryConformanceOperateRememberDedupsCoreByTags); - failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); - failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector73MemoryConformanceOperateRememberEvictsArchivalFirst); - failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector74MemoryConformanceOperateSnapshotPortRoundTrip); - failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector75MemoryConformanceOperateUpdateIndexOutOfRangeError); - failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector76TurnConformanceReplayMaxIterations); - failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector77TurnConformanceReplayNoTool); - failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector78TurnConformanceReplayPermissionDenied); - failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector79TurnConformanceReplayToolFailure); - failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector80TurnConformanceReplayToolSuccess); - failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector81TurnConformanceRunAssistantToolCallsMetadata); - failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector82TurnConformanceRunAsyncToolFunction); - failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector83TurnConformanceRunBindingsInjected); - failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector84TurnConformanceRunCancellationBeforeLlm); - failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector85TurnConformanceRunCancellationBetweenIterations); - failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector86TurnConformanceRunCancellationBetweenTools); - failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector87TurnConformanceRunContextNoTrimWhenFits); - failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector88TurnConformanceRunContextPreservesSystemMessages); - failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector89TurnConformanceRunContextTrimBasic); - failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector90TurnConformanceRunEmptyToolResult); - failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector91TurnConformanceRunEventsBasicToolLoop); - failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector92TurnConformanceRunEventsErrorLogged); - failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector93TurnConformanceRunEventsNoTools); - failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector94TurnConformanceRunGuardrailAllPass); - failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector95TurnConformanceRunGuardrailInputDeny); - failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector96TurnConformanceRunGuardrailOutputDeny); - failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector97TurnConformanceRunGuardrailToolDeny); - failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector98TurnConformanceRunMaxIterationsExceeded); - failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector99TurnConformanceRunMultiTurnToolCalls); - failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector100TurnConformanceRunMultipleToolCallsSingleTurn); - failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector101TurnConformanceRunNoToolCalls); - failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector102TurnConformanceRunParallelToolsBasic); - failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector103TurnConformanceRunParallelToolsWithGuardrailDeny); - failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector104TurnConformanceRunParallelToolsWithToolError); - failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector105TurnConformanceRunSingleToolCall); - failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector106TurnConformanceRunSteeringInjectMessage); - failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector107TurnConformanceRunSteeringMultipleMessages); - failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector108TurnConformanceRunToolNotRegisteredError); - failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector109TurnConformanceRunToolResultMessageFormat); - failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector110TurnConformanceRunTurnCancelBeforeContext); - failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector111TurnConformanceRunTurnDelegatedProviderState); - failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector112TurnConformanceRunTurnFinalOutput); - failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector113TurnConformanceRunTurnMemoryContextIsModelVisible); - failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector114TurnConformanceRunTurnOrderedToolRound); - failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector115TurnConformanceRunTurnPermissionDenialIsModelVisible); - failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector116WireConformanceToRequestAnthropicImageFormat); - failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector117WireConformanceToRequestAnthropicMaxTokensRequired); - failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector118WireConformanceToRequestAnthropicOptions); - failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector119WireConformanceToRequestAnthropicSystemSeparate); - failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector120WireConformanceToRequestAnthropicToolWire); - failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector121WireConformanceToRequestChatAudioMp3); - failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector122WireConformanceToRequestChatAudioPart); - failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector123WireConformanceToRequestChatImageBase64); - failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector124WireConformanceToRequestChatImagePart); - failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector125WireConformanceToRequestChatMultipartContent); - failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector126WireConformanceToRequestChatSimple); - failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector127WireConformanceToRequestChatSingleTextOptimized); - failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector128WireConformanceToRequestChatWithOptions); - failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector129WireConformanceToRequestEmbeddingWire); - failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector130WireConformanceToRequestImageWire); - failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector131WireConformanceToRequestKindToJsonTypeMapping); - failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector132WireConformanceToRequestOptionsAdditionalProperties); - failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector133WireConformanceToRequestOptionsMaxCompletionTokens); - failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector134WireConformanceToRequestOptionsStopSequences); - failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector135WireConformanceToRequestResponsesSimple); - failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector136WireConformanceToRequestResponsesStructuredOutput); - failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector137WireConformanceToRequestResponsesWithTools); - failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector138WireConformanceToRequestStructuredOutput); - failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector139WireConformanceToRequestStructuredOutputNestedOptional); - failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector140WireConformanceToRequestToolsBindingsStripped); - failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector141WireConformanceToRequestToolsFunctionWire); - failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector142WireConformanceToRequestToolsNullWhenEmpty); - failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector143WireConformanceToRequestToolsStrictMode); + failed += runCase("LoadConformance.load:trusted_endpoint_default_port_allowed", VectorConformanceTests::vector49LoadConformanceLoadTrustedEndpointDefaultPortAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_env_endpoint_allowed", VectorConformanceTests::vector50LoadConformanceLoadTrustedEndpointEnvEndpointAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_env_list_allowed", VectorConformanceTests::vector51LoadConformanceLoadTrustedEndpointEnvListAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_literal_key_allowed", VectorConformanceTests::vector52LoadConformanceLoadTrustedEndpointLiteralKeyAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_option_allowed", VectorConformanceTests::vector53LoadConformanceLoadTrustedEndpointOptionAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_reference_skipped", VectorConformanceTests::vector54LoadConformanceLoadTrustedEndpointReferenceSkipped); + failed += runCase("LoadConformance.load:trusted_endpoint_userinfo_allowed", VectorConformanceTests::vector55LoadConformanceLoadTrustedEndpointUserinfoAllowed); + failed += runCase("LoadConformance.load:untrusted_endpoint_ambient_rejected", VectorConformanceTests::vector56LoadConformanceLoadUntrustedEndpointAmbientRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_backslash_rejected", VectorConformanceTests::vector57LoadConformanceLoadUntrustedEndpointBackslashRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_env_default_rejected", VectorConformanceTests::vector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_env_key_rejected", VectorConformanceTests::vector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_file_connection_rejected", VectorConformanceTests::vector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_fragment_before_at_rejected", VectorConformanceTests::vector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_ipv6_literal_rejected", VectorConformanceTests::vector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", VectorConformanceTests::vector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", VectorConformanceTests::vector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_no_key_rejected", VectorConformanceTests::vector65LoadConformanceLoadUntrustedEndpointNoKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_percent_encoded_host_rejected", VectorConformanceTests::vector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_port_mismatch_rejected", VectorConformanceTests::vector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_query_before_at_rejected", VectorConformanceTests::vector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_scheme_mismatch_rejected", VectorConformanceTests::vector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_lookalike_rejected", VectorConformanceTests::vector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", VectorConformanceTests::vector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_trailing_dot_rejected", VectorConformanceTests::vector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_unicode_homoglyph_rejected", VectorConformanceTests::vector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_userinfo_lookalike_rejected", VectorConformanceTests::vector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", VectorConformanceTests::vector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_boundary_rejected", VectorConformanceTests::vector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected); + failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector77MemoryConformanceOperateClearByCategoryAndAll); + failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector78MemoryConformanceOperateFormatCoreAndRecallResults); + failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector79MemoryConformanceOperateRecallEmptyQueryAndLimit); + failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector80MemoryConformanceOperateRecallRanksByWeightedScore); + failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); + failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector82MemoryConformanceOperateRecallTagsAndCoreBoost); + failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); + failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector84MemoryConformanceOperateRememberDedupsCoreByTags); + failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); + failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector86MemoryConformanceOperateRememberEvictsArchivalFirst); + failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector87MemoryConformanceOperateSnapshotPortRoundTrip); + failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector88MemoryConformanceOperateUpdateIndexOutOfRangeError); + failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector89TurnConformanceReplayMaxIterations); + failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector90TurnConformanceReplayNoTool); + failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector91TurnConformanceReplayPermissionDenied); + failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector92TurnConformanceReplayToolFailure); + failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector93TurnConformanceReplayToolSuccess); + failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector94TurnConformanceRunAssistantToolCallsMetadata); + failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector95TurnConformanceRunAsyncToolFunction); + failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector96TurnConformanceRunBindingsInjected); + failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector97TurnConformanceRunCancellationBeforeLlm); + failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector98TurnConformanceRunCancellationBetweenIterations); + failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector99TurnConformanceRunCancellationBetweenTools); + failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector100TurnConformanceRunContextNoTrimWhenFits); + failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector101TurnConformanceRunContextPreservesSystemMessages); + failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector102TurnConformanceRunContextTrimBasic); + failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector103TurnConformanceRunEmptyToolResult); + failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector104TurnConformanceRunEventsBasicToolLoop); + failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector105TurnConformanceRunEventsErrorLogged); + failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector106TurnConformanceRunEventsNoTools); + failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector107TurnConformanceRunGuardrailAllPass); + failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector108TurnConformanceRunGuardrailInputDeny); + failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector109TurnConformanceRunGuardrailOutputDeny); + failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector110TurnConformanceRunGuardrailToolDeny); + failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector111TurnConformanceRunMaxIterationsExceeded); + failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector112TurnConformanceRunMultiTurnToolCalls); + failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector113TurnConformanceRunMultipleToolCallsSingleTurn); + failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector114TurnConformanceRunNoToolCalls); + failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector115TurnConformanceRunParallelToolsBasic); + failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector116TurnConformanceRunParallelToolsWithGuardrailDeny); + failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector117TurnConformanceRunParallelToolsWithToolError); + failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector118TurnConformanceRunSingleToolCall); + failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector119TurnConformanceRunSteeringInjectMessage); + failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector120TurnConformanceRunSteeringMultipleMessages); + failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector121TurnConformanceRunToolNotRegisteredError); + failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector122TurnConformanceRunToolResultMessageFormat); + failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector123TurnConformanceRunTurnCancelBeforeContext); + failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector124TurnConformanceRunTurnDelegatedProviderState); + failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector125TurnConformanceRunTurnFinalOutput); + failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector126TurnConformanceRunTurnMemoryContextIsModelVisible); + failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector127TurnConformanceRunTurnOrderedToolRound); + failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector128TurnConformanceRunTurnPermissionDenialIsModelVisible); + failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector129WireConformanceToRequestAnthropicImageFormat); + failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector130WireConformanceToRequestAnthropicMaxTokensRequired); + failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector131WireConformanceToRequestAnthropicOptions); + failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector132WireConformanceToRequestAnthropicSystemSeparate); + failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector133WireConformanceToRequestAnthropicToolWire); + failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector134WireConformanceToRequestChatAudioMp3); + failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector135WireConformanceToRequestChatAudioPart); + failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector136WireConformanceToRequestChatImageBase64); + failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector137WireConformanceToRequestChatImagePart); + failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector138WireConformanceToRequestChatMultipartContent); + failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector139WireConformanceToRequestChatSimple); + failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector140WireConformanceToRequestChatSingleTextOptimized); + failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector141WireConformanceToRequestChatWithOptions); + failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector142WireConformanceToRequestEmbeddingWire); + failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector143WireConformanceToRequestImageWire); + failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector144WireConformanceToRequestKindToJsonTypeMapping); + failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector145WireConformanceToRequestOptionsAdditionalProperties); + failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector146WireConformanceToRequestOptionsMaxCompletionTokens); + failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector147WireConformanceToRequestOptionsStopSequences); + failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector148WireConformanceToRequestResponsesSimple); + failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector149WireConformanceToRequestResponsesStructuredOutput); + failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector150WireConformanceToRequestResponsesWithTools); + failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector151WireConformanceToRequestStructuredOutput); + failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector152WireConformanceToRequestStructuredOutputNestedOptional); + failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector153WireConformanceToRequestToolsBindingsStripped); + failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector154WireConformanceToRequestToolsFunctionWire); + failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector155WireConformanceToRequestToolsNullWhenEmpty); + failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector156WireConformanceToRequestToolsStrictMode); if (failed > 0) { throw new AssertionError(failed + " @vector conformance failure(s)"); } diff --git a/runtime/python/prompty/prompty/model/vector_conformance.py b/runtime/python/prompty/prompty/model/vector_conformance.py index 12b28c1ab..1d4cf7de8 100644 --- a/runtime/python/prompty/prompty/model/vector_conformance.py +++ b/runtime/python/prompty/prompty/model/vector_conformance.py @@ -470,6 +470,15 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( "trusted_endpoint_builtin_allowed misrouted" ) + # vector: trusted_endpoint_default_port_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"foundry","endpoint":"https://API.OpenAI.com:443/v1"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_default_port_allowed misrouted" + ) # vector: trusted_endpoint_env_endpoint_allowed source = json.loads("null") result = seam.load(source) @@ -517,6 +526,15 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( "trusted_endpoint_reference_skipped misrouted" ) + # vector: trusted_endpoint_userinfo_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"foundry","endpoint":"https://user@api.openai.com/v1"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_userinfo_allowed misrouted" + ) # vector: untrusted_endpoint_ambient_rejected source = json.loads("null") caught = None @@ -567,6 +585,26 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: except Exception as error: # noqa: BLE001 caught = error assert caught is not None, "untrusted_endpoint_file_connection_rejected: expected an error" + # vector: untrusted_endpoint_fragment_before_at_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_fragment_before_at_rejected: expected an error" + # vector: untrusted_endpoint_ipv6_literal_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_ipv6_literal_rejected: expected an error" # vector: untrusted_endpoint_key_with_client_secret_rejected source = json.loads("null") caught = None @@ -597,6 +635,56 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: except Exception as error: # noqa: BLE001 caught = error assert caught is not None, "untrusted_endpoint_no_key_rejected: expected an error" + # vector: untrusted_endpoint_percent_encoded_host_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_percent_encoded_host_rejected: expected an error" + # vector: untrusted_endpoint_port_mismatch_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_port_mismatch_rejected: expected an error" + # vector: untrusted_endpoint_query_before_at_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_query_before_at_rejected: expected an error" + # vector: untrusted_endpoint_scheme_mismatch_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_scheme_mismatch_rejected: expected an error" + # vector: untrusted_endpoint_suffix_lookalike_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_suffix_lookalike_rejected: expected an error" # vector: untrusted_endpoint_suffix_spoof_rejected source = json.loads("null") caught = None @@ -607,6 +695,36 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: except Exception as error: # noqa: BLE001 caught = error assert caught is not None, "untrusted_endpoint_suffix_spoof_rejected: expected an error" + # vector: untrusted_endpoint_trailing_dot_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_trailing_dot_rejected: expected an error" + # vector: untrusted_endpoint_unicode_homoglyph_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_unicode_homoglyph_rejected: expected an error" + # vector: untrusted_endpoint_userinfo_lookalike_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_userinfo_lookalike_rejected: expected an error" # vector: untrusted_endpoint_wildcard_apex_rejected source = json.loads("null") caught = None @@ -617,3 +735,13 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: except Exception as error: # noqa: BLE001 caught = error assert caught is not None, "untrusted_endpoint_wildcard_apex_rejected: expected an error" + # vector: untrusted_endpoint_wildcard_boundary_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_wildcard_boundary_rejected: expected an error" diff --git a/runtime/python/prompty/tests/model/test_vector_conformance.py b/runtime/python/prompty/tests/model/test_vector_conformance.py index 11709fb20..79a1b18de 100644 --- a/runtime/python/prompty/tests/model/test_vector_conformance.py +++ b/runtime/python/prompty/tests/model/test_vector_conformance.py @@ -1734,7 +1734,40 @@ async def test_vector_48_loadconformance_load_trusted_endpoint_builtin_allowed() await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_49_loadconformance_load_trusted_endpoint_env_endpoint_allowed(): +async def test_vector_49_loadconformance_load_trusted_endpoint_default_port_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_default_port_allowed", + "description": "An explicit default port normalizes to the built-in origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-default-port-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_50_loadconformance_load_trusted_endpoint_env_endpoint_allowed(): vector_json = r""" { "name": "trusted_endpoint_env_endpoint_allowed", @@ -1770,7 +1803,7 @@ async def test_vector_49_loadconformance_load_trusted_endpoint_env_endpoint_allo await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_50_loadconformance_load_trusted_endpoint_env_list_allowed(): +async def test_vector_51_loadconformance_load_trusted_endpoint_env_list_allowed(): vector_json = r""" { "name": "trusted_endpoint_env_list_allowed", @@ -1806,7 +1839,7 @@ async def test_vector_50_loadconformance_load_trusted_endpoint_env_list_allowed( await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_51_loadconformance_load_trusted_endpoint_literal_key_allowed(): +async def test_vector_52_loadconformance_load_trusted_endpoint_literal_key_allowed(): vector_json = r""" { "name": "trusted_endpoint_literal_key_allowed", @@ -1840,7 +1873,7 @@ async def test_vector_51_loadconformance_load_trusted_endpoint_literal_key_allow await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_52_loadconformance_load_trusted_endpoint_option_allowed(): +async def test_vector_53_loadconformance_load_trusted_endpoint_option_allowed(): vector_json = r""" { "name": "trusted_endpoint_option_allowed", @@ -1876,7 +1909,7 @@ async def test_vector_52_loadconformance_load_trusted_endpoint_option_allowed(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_53_loadconformance_load_trusted_endpoint_reference_skipped(): +async def test_vector_54_loadconformance_load_trusted_endpoint_reference_skipped(): vector_json = r""" { "name": "trusted_endpoint_reference_skipped", @@ -1909,7 +1942,40 @@ async def test_vector_53_loadconformance_load_trusted_endpoint_reference_skipped await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_54_loadconformance_load_untrusted_endpoint_ambient_rejected(): +async def test_vector_55_loadconformance_load_trusted_endpoint_userinfo_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_userinfo_allowed", + "description": "Userinfo is ignored; the destination host api.openai.com is trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-userinfo-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_56_loadconformance_load_untrusted_endpoint_ambient_rejected(): vector_json = r""" { "name": "untrusted_endpoint_ambient_rejected", @@ -1938,7 +2004,7 @@ async def test_vector_54_loadconformance_load_untrusted_endpoint_ambient_rejecte await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_55_loadconformance_load_untrusted_endpoint_backslash_rejected(): +async def test_vector_57_loadconformance_load_untrusted_endpoint_backslash_rejected(): vector_json = r""" { "name": "untrusted_endpoint_backslash_rejected", @@ -1966,7 +2032,7 @@ async def test_vector_55_loadconformance_load_untrusted_endpoint_backslash_rejec await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_56_loadconformance_load_untrusted_endpoint_env_default_rejected(): +async def test_vector_58_loadconformance_load_untrusted_endpoint_env_default_rejected(): vector_json = r""" { "name": "untrusted_endpoint_env_default_rejected", @@ -1995,7 +2061,7 @@ async def test_vector_56_loadconformance_load_untrusted_endpoint_env_default_rej await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_57_loadconformance_load_untrusted_endpoint_env_key_rejected(): +async def test_vector_59_loadconformance_load_untrusted_endpoint_env_key_rejected(): vector_json = r""" { "name": "untrusted_endpoint_env_key_rejected", @@ -2027,7 +2093,7 @@ async def test_vector_57_loadconformance_load_untrusted_endpoint_env_key_rejecte await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_58_loadconformance_load_untrusted_endpoint_file_connection_rejected(): +async def test_vector_60_loadconformance_load_untrusted_endpoint_file_connection_rejected(): vector_json = r""" { "name": "untrusted_endpoint_file_connection_rejected", @@ -2058,7 +2124,63 @@ async def test_vector_58_loadconformance_load_untrusted_endpoint_file_connection await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_59_loadconformance_load_untrusted_endpoint_key_with_client_secret_rejected(): +async def test_vector_61_loadconformance_load_untrusted_endpoint_fragment_before_at_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_fragment_before_at_rejected", + "description": "A '#' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-fragment-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com#@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_62_loadconformance_load_untrusted_endpoint_ipv6_literal_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_ipv6_literal_rejected", + "description": "An IPv6 literal is not a trusted origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ipv6-literal", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://[::1]:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_63_loadconformance_load_untrusted_endpoint_key_with_client_secret_rejected(): vector_json = r""" { "name": "untrusted_endpoint_key_with_client_secret_rejected", @@ -2087,7 +2209,7 @@ async def test_vector_59_loadconformance_load_untrusted_endpoint_key_with_client await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_60_loadconformance_load_untrusted_endpoint_mismatched_credential_rejected(): +async def test_vector_64_loadconformance_load_untrusted_endpoint_mismatched_credential_rejected(): vector_json = r""" { "name": "untrusted_endpoint_mismatched_credential_rejected", @@ -2116,7 +2238,7 @@ async def test_vector_60_loadconformance_load_untrusted_endpoint_mismatched_cred await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_61_loadconformance_load_untrusted_endpoint_no_key_rejected(): +async def test_vector_65_loadconformance_load_untrusted_endpoint_no_key_rejected(): vector_json = r""" { "name": "untrusted_endpoint_no_key_rejected", @@ -2144,7 +2266,147 @@ async def test_vector_61_loadconformance_load_untrusted_endpoint_no_key_rejected await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_62_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected(): +async def test_vector_66_loadconformance_load_untrusted_endpoint_percent_encoded_host_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_percent_encoded_host_rejected", + "description": "Hosts are compared without percent-decoding, so an encoded host never matches", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-percent-encoded-host", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com%2eevil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_67_loadconformance_load_untrusted_endpoint_port_mismatch_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_port_mismatch_rejected", + "description": "A non-default port is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-port-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com:8443" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_68_loadconformance_load_untrusted_endpoint_query_before_at_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_query_before_at_rejected", + "description": "A '?' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-query-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com?@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_69_loadconformance_load_untrusted_endpoint_scheme_mismatch_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_scheme_mismatch_rejected", + "description": "Built-in origins are https; http on the same host is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-scheme-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_70_loadconformance_load_untrusted_endpoint_suffix_lookalike_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_suffix_lookalike_rejected", + "description": "A trusted host as a label prefix of another domain is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-suffix-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com.evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_71_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected(): vector_json = r""" { "name": "untrusted_endpoint_suffix_spoof_rejected", @@ -2175,7 +2437,91 @@ async def test_vector_62_loadconformance_load_untrusted_endpoint_suffix_spoof_re await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_63_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected(): +async def test_vector_72_loadconformance_load_untrusted_endpoint_trailing_dot_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_trailing_dot_rejected", + "description": "Hosts are compared exactly; a trailing-dot host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-trailing-dot", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com." + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_73_loadconformance_load_untrusted_endpoint_unicode_homoglyph_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_unicode_homoglyph_rejected", + "description": "A non-ASCII lookalike host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-unicode-homoglyph", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://аpi.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_74_loadconformance_load_untrusted_endpoint_userinfo_lookalike_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_userinfo_lookalike_rejected", + "description": "A trusted host in userinfo is not the destination host", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-userinfo-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com@evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_75_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected(): vector_json = r""" { "name": "untrusted_endpoint_wildcard_apex_rejected", @@ -2203,7 +2549,35 @@ async def test_vector_63_loadconformance_load_untrusted_endpoint_wildcard_apex_r await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_64_memoryconformance_operate_clear_by_category_and_all(): +async def test_vector_76_loadconformance_load_untrusted_endpoint_wildcard_boundary_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_wildcard_boundary_rejected", + "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-wildcard-boundary", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evilservices.ai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_77_memoryconformance_operate_clear_by_category_and_all(): vector_json = r""" { "name": "clear_by_category_and_all", @@ -2255,7 +2629,7 @@ async def test_vector_64_memoryconformance_operate_clear_by_category_and_all(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_65_memoryconformance_operate_format_core_and_recall_results(): +async def test_vector_78_memoryconformance_operate_format_core_and_recall_results(): vector_json = r""" { "name": "format_core_and_recall_results", @@ -2295,7 +2669,7 @@ async def test_vector_65_memoryconformance_operate_format_core_and_recall_result await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_66_memoryconformance_operate_recall_empty_query_and_limit(): +async def test_vector_79_memoryconformance_operate_recall_empty_query_and_limit(): vector_json = r""" { "name": "recall_empty_query_and_limit", @@ -2348,7 +2722,7 @@ async def test_vector_66_memoryconformance_operate_recall_empty_query_and_limit( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_67_memoryconformance_operate_recall_ranks_by_weighted_score(): +async def test_vector_80_memoryconformance_operate_recall_ranks_by_weighted_score(): vector_json = r""" { "name": "recall_ranks_by_weighted_score", @@ -2405,7 +2779,7 @@ async def test_vector_67_memoryconformance_operate_recall_ranks_by_weighted_scor await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_68_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): +async def test_vector_81_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): vector_json = r""" { "name": "recall_score_ties_preserve_insertion_order", @@ -2461,7 +2835,7 @@ async def test_vector_68_memoryconformance_operate_recall_score_ties_preserve_in await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_69_memoryconformance_operate_recall_tags_and_core_boost(): +async def test_vector_82_memoryconformance_operate_recall_tags_and_core_boost(): vector_json = r""" { "name": "recall_tags_and_core_boost", @@ -2520,7 +2894,7 @@ async def test_vector_69_memoryconformance_operate_recall_tags_and_core_boost(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_70_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): +async def test_vector_83_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): vector_json = r""" { "name": "recall_tokenization_punctuation_unicode_and_dedup", @@ -2579,7 +2953,7 @@ async def test_vector_70_memoryconformance_operate_recall_tokenization_punctuati await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_71_memoryconformance_operate_remember_dedups_core_by_tags(): +async def test_vector_84_memoryconformance_operate_remember_dedups_core_by_tags(): vector_json = r""" { "name": "remember_dedups_core_by_tags", @@ -2641,7 +3015,7 @@ async def test_vector_71_memoryconformance_operate_remember_dedups_core_by_tags( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_72_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): +async def test_vector_85_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): vector_json = r""" { "name": "remember_dedups_core_missing_and_empty_tags", @@ -2695,7 +3069,7 @@ async def test_vector_72_memoryconformance_operate_remember_dedups_core_missing_ await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_73_memoryconformance_operate_remember_evicts_archival_first(): +async def test_vector_86_memoryconformance_operate_remember_evicts_archival_first(): vector_json = r""" { "name": "remember_evicts_archival_first", @@ -2754,7 +3128,7 @@ async def test_vector_73_memoryconformance_operate_remember_evicts_archival_firs await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_74_memoryconformance_operate_snapshot_port_round_trip(): +async def test_vector_87_memoryconformance_operate_snapshot_port_round_trip(): vector_json = r""" { "name": "snapshot_port_round_trip", @@ -2792,7 +3166,7 @@ async def test_vector_74_memoryconformance_operate_snapshot_port_round_trip(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_75_memoryconformance_operate_update_index_out_of_range_error(): +async def test_vector_88_memoryconformance_operate_update_index_out_of_range_error(): vector_json = r""" { "name": "update_index_out_of_range_error", @@ -2824,7 +3198,7 @@ async def test_vector_75_memoryconformance_operate_update_index_out_of_range_err await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_76_turnconformance_replay_max_iterations(): +async def test_vector_89_turnconformance_replay_max_iterations(): vector_json = r""" { "name": "max_iterations", @@ -2859,7 +3233,7 @@ async def test_vector_76_turnconformance_replay_max_iterations(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_77_turnconformance_replay_no_tool(): +async def test_vector_90_turnconformance_replay_no_tool(): vector_json = r""" { "name": "no_tool", @@ -2890,7 +3264,7 @@ async def test_vector_77_turnconformance_replay_no_tool(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_78_turnconformance_replay_permission_denied(): +async def test_vector_91_turnconformance_replay_permission_denied(): vector_json = r""" { "name": "permission_denied", @@ -2924,7 +3298,7 @@ async def test_vector_78_turnconformance_replay_permission_denied(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_79_turnconformance_replay_tool_failure(): +async def test_vector_92_turnconformance_replay_tool_failure(): vector_json = r""" { "name": "tool_failure", @@ -2960,7 +3334,7 @@ async def test_vector_79_turnconformance_replay_tool_failure(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_80_turnconformance_replay_tool_success(): +async def test_vector_93_turnconformance_replay_tool_success(): vector_json = r""" { "name": "tool_success", @@ -2996,7 +3370,7 @@ async def test_vector_80_turnconformance_replay_tool_success(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_81_turnconformance_run_assistant_tool_calls_metadata(): +async def test_vector_94_turnconformance_run_assistant_tool_calls_metadata(): vector_json = r""" { "name": "assistant_tool_calls_metadata", @@ -3148,7 +3522,7 @@ async def test_vector_81_turnconformance_run_assistant_tool_calls_metadata(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_82_turnconformance_run_async_tool_function(): +async def test_vector_95_turnconformance_run_async_tool_function(): vector_json = r""" { "name": "async_tool_function", @@ -3258,7 +3632,7 @@ async def test_vector_82_turnconformance_run_async_tool_function(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_83_turnconformance_run_bindings_injected(): +async def test_vector_96_turnconformance_run_bindings_injected(): vector_json = r""" { "name": "bindings_injected", @@ -3388,7 +3762,7 @@ async def test_vector_83_turnconformance_run_bindings_injected(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_84_turnconformance_run_cancellation_before_llm(): +async def test_vector_97_turnconformance_run_cancellation_before_llm(): vector_json = r""" { "name": "cancellation_before_llm", @@ -3449,7 +3823,7 @@ async def test_vector_84_turnconformance_run_cancellation_before_llm(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_85_turnconformance_run_cancellation_between_iterations(): +async def test_vector_98_turnconformance_run_cancellation_between_iterations(): vector_json = r""" { "name": "cancellation_between_iterations", @@ -3579,7 +3953,7 @@ async def test_vector_85_turnconformance_run_cancellation_between_iterations(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_86_turnconformance_run_cancellation_between_tools(): +async def test_vector_99_turnconformance_run_cancellation_between_tools(): vector_json = r""" { "name": "cancellation_between_tools", @@ -3718,7 +4092,7 @@ async def test_vector_86_turnconformance_run_cancellation_between_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_87_turnconformance_run_context_no_trim_when_fits(): +async def test_vector_100_turnconformance_run_context_no_trim_when_fits(): vector_json = r""" { "name": "context_no_trim_when_fits", @@ -3789,7 +4163,7 @@ async def test_vector_87_turnconformance_run_context_no_trim_when_fits(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_88_turnconformance_run_context_preserves_system_messages(): +async def test_vector_101_turnconformance_run_context_preserves_system_messages(): vector_json = r""" { "name": "context_preserves_system_messages", @@ -3896,7 +4270,7 @@ async def test_vector_88_turnconformance_run_context_preserves_system_messages() await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_89_turnconformance_run_context_trim_basic(): +async def test_vector_102_turnconformance_run_context_trim_basic(): vector_json = r""" { "name": "context_trim_basic", @@ -4054,7 +4428,7 @@ async def test_vector_89_turnconformance_run_context_trim_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_90_turnconformance_run_empty_tool_result(): +async def test_vector_103_turnconformance_run_empty_tool_result(): vector_json = r""" { "name": "empty_tool_result", @@ -4169,7 +4543,7 @@ async def test_vector_90_turnconformance_run_empty_tool_result(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_91_turnconformance_run_events_basic_tool_loop(): +async def test_vector_104_turnconformance_run_events_basic_tool_loop(): vector_json = r""" { "name": "events_basic_tool_loop", @@ -4315,7 +4689,7 @@ async def test_vector_91_turnconformance_run_events_basic_tool_loop(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_92_turnconformance_run_events_error_logged(): +async def test_vector_105_turnconformance_run_events_error_logged(): vector_json = r""" { "name": "events_error_logged", @@ -4441,7 +4815,7 @@ async def test_vector_92_turnconformance_run_events_error_logged(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_93_turnconformance_run_events_no_tools(): +async def test_vector_106_turnconformance_run_events_no_tools(): vector_json = r""" { "name": "events_no_tools", @@ -4519,7 +4893,7 @@ async def test_vector_93_turnconformance_run_events_no_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_94_turnconformance_run_guardrail_all_pass(): +async def test_vector_107_turnconformance_run_guardrail_all_pass(): vector_json = r""" { "name": "guardrail_all_pass", @@ -4642,7 +5016,7 @@ async def test_vector_94_turnconformance_run_guardrail_all_pass(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_95_turnconformance_run_guardrail_input_deny(): +async def test_vector_108_turnconformance_run_guardrail_input_deny(): vector_json = r""" { "name": "guardrail_input_deny", @@ -4698,7 +5072,7 @@ async def test_vector_95_turnconformance_run_guardrail_input_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_96_turnconformance_run_guardrail_output_deny(): +async def test_vector_109_turnconformance_run_guardrail_output_deny(): vector_json = r""" { "name": "guardrail_output_deny", @@ -4776,7 +5150,7 @@ async def test_vector_96_turnconformance_run_guardrail_output_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_97_turnconformance_run_guardrail_tool_deny(): +async def test_vector_110_turnconformance_run_guardrail_tool_deny(): vector_json = r""" { "name": "guardrail_tool_deny", @@ -4930,7 +5304,7 @@ async def test_vector_97_turnconformance_run_guardrail_tool_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_98_turnconformance_run_max_iterations_exceeded(): +async def test_vector_111_turnconformance_run_max_iterations_exceeded(): vector_json = r""" { "name": "max_iterations_exceeded", @@ -5440,7 +5814,7 @@ async def test_vector_98_turnconformance_run_max_iterations_exceeded(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_99_turnconformance_run_multi_turn_tool_calls(): +async def test_vector_112_turnconformance_run_multi_turn_tool_calls(): vector_json = r""" { "name": "multi_turn_tool_calls", @@ -5617,7 +5991,7 @@ async def test_vector_99_turnconformance_run_multi_turn_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_100_turnconformance_run_multiple_tool_calls_single_turn(): +async def test_vector_113_turnconformance_run_multiple_tool_calls_single_turn(): vector_json = r""" { "name": "multiple_tool_calls_single_turn", @@ -5798,7 +6172,7 @@ async def test_vector_100_turnconformance_run_multiple_tool_calls_single_turn(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_101_turnconformance_run_no_tool_calls(): +async def test_vector_114_turnconformance_run_no_tool_calls(): vector_json = r""" { "name": "no_tool_calls", @@ -5866,7 +6240,7 @@ async def test_vector_101_turnconformance_run_no_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_102_turnconformance_run_parallel_tools_basic(): +async def test_vector_115_turnconformance_run_parallel_tools_basic(): vector_json = r""" { "name": "parallel_tools_basic", @@ -6039,7 +6413,7 @@ async def test_vector_102_turnconformance_run_parallel_tools_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_103_turnconformance_run_parallel_tools_with_guardrail_deny(): +async def test_vector_116_turnconformance_run_parallel_tools_with_guardrail_deny(): vector_json = r""" { "name": "parallel_tools_with_guardrail_deny", @@ -6230,7 +6604,7 @@ async def test_vector_103_turnconformance_run_parallel_tools_with_guardrail_deny await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_104_turnconformance_run_parallel_tools_with_tool_error(): +async def test_vector_117_turnconformance_run_parallel_tools_with_tool_error(): vector_json = r""" { "name": "parallel_tools_with_tool_error", @@ -6403,7 +6777,7 @@ async def test_vector_104_turnconformance_run_parallel_tools_with_tool_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_105_turnconformance_run_single_tool_call(): +async def test_vector_118_turnconformance_run_single_tool_call(): vector_json = r""" { "name": "single_tool_call", @@ -6550,7 +6924,7 @@ async def test_vector_105_turnconformance_run_single_tool_call(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_106_turnconformance_run_steering_inject_message(): +async def test_vector_119_turnconformance_run_steering_inject_message(): vector_json = r""" { "name": "steering_inject_message", @@ -6758,7 +7132,7 @@ async def test_vector_106_turnconformance_run_steering_inject_message(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_107_turnconformance_run_steering_multiple_messages(): +async def test_vector_120_turnconformance_run_steering_multiple_messages(): vector_json = r""" { "name": "steering_multiple_messages", @@ -6928,7 +7302,7 @@ async def test_vector_107_turnconformance_run_steering_multiple_messages(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_108_turnconformance_run_tool_not_registered_error(): +async def test_vector_121_turnconformance_run_tool_not_registered_error(): vector_json = r""" { "name": "tool_not_registered_error", @@ -7012,7 +7386,7 @@ async def test_vector_108_turnconformance_run_tool_not_registered_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_109_turnconformance_run_tool_result_message_format(): +async def test_vector_122_turnconformance_run_tool_result_message_format(): vector_json = r""" { "name": "tool_result_message_format", @@ -7133,7 +7507,7 @@ async def test_vector_109_turnconformance_run_tool_result_message_format(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_110_turnconformance_runturn_cancel_before_context(): +async def test_vector_123_turnconformance_runturn_cancel_before_context(): vector_json = r""" { "name": "cancel_before_context", @@ -7165,7 +7539,7 @@ async def test_vector_110_turnconformance_runturn_cancel_before_context(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_111_turnconformance_runturn_delegated_provider_state(): +async def test_vector_124_turnconformance_runturn_delegated_provider_state(): vector_json = r""" { "name": "delegated_provider_state", @@ -7232,7 +7606,7 @@ async def test_vector_111_turnconformance_runturn_delegated_provider_state(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_112_turnconformance_runturn_final_output(): +async def test_vector_125_turnconformance_runturn_final_output(): vector_json = r""" { "name": "final_output", @@ -7277,7 +7651,7 @@ async def test_vector_112_turnconformance_runturn_final_output(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_113_turnconformance_runturn_memory_context_is_model_visible(): +async def test_vector_126_turnconformance_runturn_memory_context_is_model_visible(): vector_json = r""" { "name": "memory_context_is_model_visible", @@ -7339,7 +7713,7 @@ async def test_vector_113_turnconformance_runturn_memory_context_is_model_visibl await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_114_turnconformance_runturn_ordered_tool_round(): +async def test_vector_127_turnconformance_runturn_ordered_tool_round(): vector_json = r""" { "name": "ordered_tool_round", @@ -7430,7 +7804,7 @@ async def test_vector_114_turnconformance_runturn_ordered_tool_round(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_115_turnconformance_runturn_permission_denial_is_model_visible(): +async def test_vector_128_turnconformance_runturn_permission_denial_is_model_visible(): vector_json = r""" { "name": "permission_denial_is_model_visible", @@ -7481,7 +7855,7 @@ async def test_vector_115_turnconformance_runturn_permission_denial_is_model_vis await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_116_wireconformance_torequest_anthropic_image_format(): +async def test_vector_129_wireconformance_torequest_anthropic_image_format(): vector_json = r""" { "name": "anthropic_image_format", @@ -7551,7 +7925,7 @@ async def test_vector_116_wireconformance_torequest_anthropic_image_format(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_117_wireconformance_torequest_anthropic_max_tokens_required(): +async def test_vector_130_wireconformance_torequest_anthropic_max_tokens_required(): vector_json = r""" { "name": "anthropic_max_tokens_required", @@ -7606,7 +7980,7 @@ async def test_vector_117_wireconformance_torequest_anthropic_max_tokens_require await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_118_wireconformance_torequest_anthropic_options(): +async def test_vector_131_wireconformance_torequest_anthropic_options(): vector_json = r""" { "name": "anthropic_options", @@ -7675,7 +8049,7 @@ async def test_vector_118_wireconformance_torequest_anthropic_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_119_wireconformance_torequest_anthropic_system_separate(): +async def test_vector_132_wireconformance_torequest_anthropic_system_separate(): vector_json = r""" { "name": "anthropic_system_separate", @@ -7742,7 +8116,7 @@ async def test_vector_119_wireconformance_torequest_anthropic_system_separate(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_120_wireconformance_torequest_anthropic_tool_wire(): +async def test_vector_133_wireconformance_torequest_anthropic_tool_wire(): vector_json = r""" { "name": "anthropic_tool_wire", @@ -7827,7 +8201,7 @@ async def test_vector_120_wireconformance_torequest_anthropic_tool_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_121_wireconformance_torequest_chat_audio_mp3(): +async def test_vector_134_wireconformance_torequest_chat_audio_mp3(): vector_json = r""" { "name": "chat_audio_mp3", @@ -7885,7 +8259,7 @@ async def test_vector_121_wireconformance_torequest_chat_audio_mp3(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_122_wireconformance_torequest_chat_audio_part(): +async def test_vector_135_wireconformance_torequest_chat_audio_part(): vector_json = r""" { "name": "chat_audio_part", @@ -7943,7 +8317,7 @@ async def test_vector_122_wireconformance_torequest_chat_audio_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_123_wireconformance_torequest_chat_image_base64(): +async def test_vector_136_wireconformance_torequest_chat_image_base64(): vector_json = r""" { "name": "chat_image_base64", @@ -8007,7 +8381,7 @@ async def test_vector_123_wireconformance_torequest_chat_image_base64(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_124_wireconformance_torequest_chat_image_part(): +async def test_vector_137_wireconformance_torequest_chat_image_part(): vector_json = r""" { "name": "chat_image_part", @@ -8071,7 +8445,7 @@ async def test_vector_124_wireconformance_torequest_chat_image_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_125_wireconformance_torequest_chat_multipart_content(): +async def test_vector_138_wireconformance_torequest_chat_multipart_content(): vector_json = r""" { "name": "chat_multipart_content", @@ -8135,7 +8509,7 @@ async def test_vector_125_wireconformance_torequest_chat_multipart_content(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_126_wireconformance_torequest_chat_simple(): +async def test_vector_139_wireconformance_torequest_chat_simple(): vector_json = r""" { "name": "chat_simple", @@ -8197,7 +8571,7 @@ async def test_vector_126_wireconformance_torequest_chat_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_127_wireconformance_torequest_chat_single_text_optimized(): +async def test_vector_140_wireconformance_torequest_chat_single_text_optimized(): vector_json = r""" { "name": "chat_single_text_optimized", @@ -8246,7 +8620,7 @@ async def test_vector_127_wireconformance_torequest_chat_single_text_optimized() await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_128_wireconformance_torequest_chat_with_options(): +async def test_vector_141_wireconformance_torequest_chat_with_options(): vector_json = r""" { "name": "chat_with_options", @@ -8314,7 +8688,7 @@ async def test_vector_128_wireconformance_torequest_chat_with_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_129_wireconformance_torequest_embedding_wire(): +async def test_vector_142_wireconformance_torequest_embedding_wire(): vector_json = r""" { "name": "embedding_wire", @@ -8358,7 +8732,7 @@ async def test_vector_129_wireconformance_torequest_embedding_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_130_wireconformance_torequest_image_wire(): +async def test_vector_143_wireconformance_torequest_image_wire(): vector_json = r""" { "name": "image_wire", @@ -8402,7 +8776,7 @@ async def test_vector_130_wireconformance_torequest_image_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_131_wireconformance_torequest_kind_to_json_type_mapping(): +async def test_vector_144_wireconformance_torequest_kind_to_json_type_mapping(): vector_json = r""" { "name": "kind_to_json_type_mapping", @@ -8521,7 +8895,7 @@ async def test_vector_131_wireconformance_torequest_kind_to_json_type_mapping(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_132_wireconformance_torequest_options_additional_properties(): +async def test_vector_145_wireconformance_torequest_options_additional_properties(): vector_json = r""" { "name": "options_additional_properties", @@ -8575,7 +8949,7 @@ async def test_vector_132_wireconformance_torequest_options_additional_propertie await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_133_wireconformance_torequest_options_max_completion_tokens(): +async def test_vector_146_wireconformance_torequest_options_max_completion_tokens(): vector_json = r""" { "name": "options_max_completion_tokens", @@ -8627,7 +9001,7 @@ async def test_vector_133_wireconformance_torequest_options_max_completion_token await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_134_wireconformance_torequest_options_stop_sequences(): +async def test_vector_147_wireconformance_torequest_options_stop_sequences(): vector_json = r""" { "name": "options_stop_sequences", @@ -8685,7 +9059,7 @@ async def test_vector_134_wireconformance_torequest_options_stop_sequences(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_135_wireconformance_torequest_responses_simple(): +async def test_vector_148_wireconformance_torequest_responses_simple(): vector_json = r""" { "name": "responses_simple", @@ -8744,7 +9118,7 @@ async def test_vector_135_wireconformance_torequest_responses_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_136_wireconformance_torequest_responses_structured_output(): +async def test_vector_149_wireconformance_torequest_responses_structured_output(): vector_json = r""" { "name": "responses_structured_output", @@ -8831,7 +9205,7 @@ async def test_vector_136_wireconformance_torequest_responses_structured_output( await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_137_wireconformance_torequest_responses_with_tools(): +async def test_vector_150_wireconformance_torequest_responses_with_tools(): vector_json = r""" { "name": "responses_with_tools", @@ -8911,7 +9285,7 @@ async def test_vector_137_wireconformance_torequest_responses_with_tools(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_138_wireconformance_torequest_structured_output(): +async def test_vector_151_wireconformance_torequest_structured_output(): vector_json = r""" { "name": "structured_output", @@ -8994,7 +9368,7 @@ async def test_vector_138_wireconformance_torequest_structured_output(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_139_wireconformance_torequest_structured_output_nested_optional(): +async def test_vector_152_wireconformance_torequest_structured_output_nested_optional(): vector_json = r""" { "name": "structured_output_nested_optional", @@ -9096,7 +9470,7 @@ async def test_vector_139_wireconformance_torequest_structured_output_nested_opt await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_140_wireconformance_torequest_tools_bindings_stripped(): +async def test_vector_153_wireconformance_torequest_tools_bindings_stripped(): vector_json = r""" { "name": "tools_bindings_stripped", @@ -9188,7 +9562,7 @@ async def test_vector_140_wireconformance_torequest_tools_bindings_stripped(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_141_wireconformance_torequest_tools_function_wire(): +async def test_vector_154_wireconformance_torequest_tools_function_wire(): vector_json = r""" { "name": "tools_function_wire", @@ -9270,7 +9644,7 @@ async def test_vector_141_wireconformance_torequest_tools_function_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_142_wireconformance_torequest_tools_null_when_empty(): +async def test_vector_155_wireconformance_torequest_tools_null_when_empty(): vector_json = r""" { "name": "tools_null_when_empty", @@ -9319,7 +9693,7 @@ async def test_vector_142_wireconformance_torequest_tools_null_when_empty(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_143_wireconformance_torequest_tools_strict_mode(): +async def test_vector_156_wireconformance_torequest_tools_strict_mode(): vector_json = r""" { "name": "tools_strict_mode", diff --git a/runtime/rust/prompty/src/model/vector_conformance.rs b/runtime/rust/prompty/src/model/vector_conformance.rs index 331462f72..6c80c5106 100644 --- a/runtime/rust/prompty/src/model/vector_conformance.rs +++ b/runtime/rust/prompty/src/model/vector_conformance.rs @@ -1386,6 +1386,37 @@ pub async fn run_load_conformance_conformance(_ seam: S) async t throw SeamConformanceError("trusted_endpoint_builtin_allowed misrouted") } } + // vector: trusted_endpoint_default_port_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-default-port-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_default_port_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_default_port_allowed misrouted") + } + } // vector: trusted_endpoint_env_endpoint_allowed do { let inputData = "{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}}".data(using: .utf8)! @@ -754,6 +768,20 @@ public func runLoadConformanceConformance(_ seam: S) async t throw SeamConformanceError("trusted_endpoint_reference_skipped misrouted") } } + // vector: trusted_endpoint_userinfo_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-userinfo-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_userinfo_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_userinfo_allowed misrouted") + } + } // vector: untrusted_endpoint_ambient_rejected do { let inputData = "{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}}".data(using: .utf8)! @@ -839,6 +867,40 @@ public func runLoadConformanceConformance(_ seam: S) async t throw SeamConformanceError("untrusted_endpoint_file_connection_rejected: expected an error") } } + // vector: untrusted_endpoint_fragment_before_at_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_fragment_before_at_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_fragment_before_at_rejected: expected an error") + } + } + // vector: untrusted_endpoint_ipv6_literal_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_ipv6_literal_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_ipv6_literal_rejected: expected an error") + } + } // vector: untrusted_endpoint_key_with_client_secret_rejected do { let inputData = "{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}}".data(using: .utf8)! @@ -890,6 +952,91 @@ public func runLoadConformanceConformance(_ seam: S) async t throw SeamConformanceError("untrusted_endpoint_no_key_rejected: expected an error") } } + // vector: untrusted_endpoint_percent_encoded_host_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_percent_encoded_host_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_percent_encoded_host_rejected: expected an error") + } + } + // vector: untrusted_endpoint_port_mismatch_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_port_mismatch_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_port_mismatch_rejected: expected an error") + } + } + // vector: untrusted_endpoint_query_before_at_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_query_before_at_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_query_before_at_rejected: expected an error") + } + } + // vector: untrusted_endpoint_scheme_mismatch_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_scheme_mismatch_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_scheme_mismatch_rejected: expected an error") + } + } + // vector: untrusted_endpoint_suffix_lookalike_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_suffix_lookalike_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_suffix_lookalike_rejected: expected an error") + } + } // vector: untrusted_endpoint_suffix_spoof_rejected do { let inputData = "{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]}".data(using: .utf8)! @@ -907,6 +1054,57 @@ public func runLoadConformanceConformance(_ seam: S) async t throw SeamConformanceError("untrusted_endpoint_suffix_spoof_rejected: expected an error") } } + // vector: untrusted_endpoint_trailing_dot_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_trailing_dot_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_trailing_dot_rejected: expected an error") + } + } + // vector: untrusted_endpoint_unicode_homoglyph_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\u{430}pi.openai.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_unicode_homoglyph_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_unicode_homoglyph_rejected: expected an error") + } + } + // vector: untrusted_endpoint_userinfo_lookalike_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_userinfo_lookalike_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_userinfo_lookalike_rejected: expected an error") + } + } // vector: untrusted_endpoint_wildcard_apex_rejected do { let inputData = "{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}}".data(using: .utf8)! @@ -924,4 +1122,21 @@ public func runLoadConformanceConformance(_ seam: S) async t throw SeamConformanceError("untrusted_endpoint_wildcard_apex_rejected: expected an error") } } + // vector: untrusted_endpoint_wildcard_boundary_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_wildcard_boundary_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_wildcard_boundary_rejected: expected an error") + } + } } diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift index 7e65efd4e..9277913b1 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift @@ -524,7 +524,17 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector49LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() async throws { + func testVector49LoadConformanceLoadTrustedEndpointDefaultPortAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_default_port_allowed\",\"description\":\"An explicit default port normalizes to the built-in origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-default-port-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector50LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() async throws { guard let vectorData = "{\"name\":\"trusted_endpoint_env_endpoint_allowed\",\"description\":\"An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -534,7 +544,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector50LoadConformanceLoadTrustedEndpointEnvListAllowed() async throws { + func testVector51LoadConformanceLoadTrustedEndpointEnvListAllowed() async throws { guard let vectorData = "{\"name\":\"trusted_endpoint_env_list_allowed\",\"description\":\"PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -544,7 +554,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector51LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() async throws { + func testVector52LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() async throws { guard let vectorData = "{\"name\":\"trusted_endpoint_literal_key_allowed\",\"description\":\"A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -554,7 +564,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector52LoadConformanceLoadTrustedEndpointOptionAllowed() async throws { + func testVector53LoadConformanceLoadTrustedEndpointOptionAllowed() async throws { guard let vectorData = "{\"name\":\"trusted_endpoint_option_allowed\",\"description\":\"A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -564,7 +574,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector53LoadConformanceLoadTrustedEndpointReferenceSkipped() async throws { + func testVector54LoadConformanceLoadTrustedEndpointReferenceSkipped() async throws { guard let vectorData = "{\"name\":\"trusted_endpoint_reference_skipped\",\"description\":\"kind: reference connections carry no endpoint and are not subject to the trust check\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -574,7 +584,17 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector54LoadConformanceLoadUntrustedEndpointAmbientRejected() async throws { + func testVector55LoadConformanceLoadTrustedEndpointUserinfoAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_userinfo_allowed\",\"description\":\"Userinfo is ignored; the destination host api.openai.com is trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-userinfo-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector56LoadConformanceLoadUntrustedEndpointAmbientRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_ambient_rejected\",\"description\":\"A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec \u{a7}2.5.1)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -584,7 +604,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector55LoadConformanceLoadUntrustedEndpointBackslashRejected() async throws { + func testVector57LoadConformanceLoadUntrustedEndpointBackslashRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_backslash_rejected\",\"description\":\"A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -594,7 +614,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector56LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() async throws { + func testVector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -604,7 +624,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector57LoadConformanceLoadUntrustedEndpointEnvKeyRejected() async throws { + func testVector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -614,7 +634,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector58LoadConformanceLoadUntrustedEndpointFileConnectionRejected() async throws { + func testVector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -624,7 +644,27 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector59LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() async throws { + func testVector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_fragment_before_at_rejected\",\"description\":\"A '#' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_ipv6_literal_rejected\",\"description\":\"An IPv6 literal is not a trusted origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -634,7 +674,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector60LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() async throws { + func testVector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -644,7 +684,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector61LoadConformanceLoadUntrustedEndpointNoKeyRejected() async throws { + func testVector65LoadConformanceLoadUntrustedEndpointNoKeyRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -654,7 +694,57 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector62LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() async throws { + func testVector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_percent_encoded_host_rejected\",\"description\":\"Hosts are compared without percent-decoding, so an encoded host never matches\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_port_mismatch_rejected\",\"description\":\"A non-default port is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_query_before_at_rejected\",\"description\":\"A '?' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_scheme_mismatch_rejected\",\"description\":\"Built-in origins are https; http on the same host is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_suffix_lookalike_rejected\",\"description\":\"A trusted host as a label prefix of another domain is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -664,7 +754,37 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector63LoadConformanceLoadUntrustedEndpointWildcardApexRejected() async throws { + func testVector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_trailing_dot_rejected\",\"description\":\"Hosts are compared exactly; a trailing-dot host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_unicode_homoglyph_rejected\",\"description\":\"A non-ASCII lookalike host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\u{430}pi.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_userinfo_lookalike_rejected\",\"description\":\"A trusted host in userinfo is not the destination host\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -674,7 +794,17 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector64MemoryConformanceOperateClearByCategoryAndAll() async throws { + func testVector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_wildcard_boundary_rejected\",\"description\":\"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector77MemoryConformanceOperateClearByCategoryAndAll() async throws { guard let vectorData = "{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -684,7 +814,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector65MemoryConformanceOperateFormatCoreAndRecallResults() async throws { + func testVector78MemoryConformanceOperateFormatCoreAndRecallResults() async throws { guard let vectorData = "{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -694,7 +824,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector66MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { + func testVector79MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { guard let vectorData = "{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -704,7 +834,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector67MemoryConformanceOperateRecallRanksByWeightedScore() async throws { + func testVector80MemoryConformanceOperateRecallRanksByWeightedScore() async throws { guard let vectorData = "{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -714,7 +844,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector68MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { + func testVector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { guard let vectorData = "{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -724,7 +854,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector69MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { + func testVector82MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { guard let vectorData = "{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -734,7 +864,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector70MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { + func testVector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { guard let vectorData = "{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\u{e9}\"]},{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\u{e9}\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -744,7 +874,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector71MemoryConformanceOperateRememberDedupsCoreByTags() async throws { + func testVector84MemoryConformanceOperateRememberDedupsCoreByTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -754,7 +884,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector72MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { + func testVector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -764,7 +894,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector73MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { + func testVector86MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { guard let vectorData = "{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -774,7 +904,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector74MemoryConformanceOperateSnapshotPortRoundTrip() async throws { + func testVector87MemoryConformanceOperateSnapshotPortRoundTrip() async throws { guard let vectorData = "{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -784,7 +914,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector75MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { + func testVector88MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { guard let vectorData = "{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -794,7 +924,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector76TurnConformanceReplayMaxIterations() async throws { + func testVector89TurnConformanceReplayMaxIterations() async throws { guard let vectorData = "{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -804,7 +934,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector77TurnConformanceReplayNoTool() async throws { + func testVector90TurnConformanceReplayNoTool() async throws { guard let vectorData = "{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -814,7 +944,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector78TurnConformanceReplayPermissionDenied() async throws { + func testVector91TurnConformanceReplayPermissionDenied() async throws { guard let vectorData = "{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -824,7 +954,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector79TurnConformanceReplayToolFailure() async throws { + func testVector92TurnConformanceReplayToolFailure() async throws { guard let vectorData = "{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -834,7 +964,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector80TurnConformanceReplayToolSuccess() async throws { + func testVector93TurnConformanceReplayToolSuccess() async throws { guard let vectorData = "{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -844,7 +974,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector81TurnConformanceRunAssistantToolCallsMetadata() async throws { + func testVector94TurnConformanceRunAssistantToolCallsMetadata() async throws { guard let vectorData = "{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -854,7 +984,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector82TurnConformanceRunAsyncToolFunction() async throws { + func testVector95TurnConformanceRunAsyncToolFunction() async throws { guard let vectorData = "{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -864,7 +994,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector83TurnConformanceRunBindingsInjected() async throws { + func testVector96TurnConformanceRunBindingsInjected() async throws { guard let vectorData = "{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -874,7 +1004,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector84TurnConformanceRunCancellationBeforeLlm() async throws { + func testVector97TurnConformanceRunCancellationBeforeLlm() async throws { guard let vectorData = "{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -884,7 +1014,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector85TurnConformanceRunCancellationBetweenIterations() async throws { + func testVector98TurnConformanceRunCancellationBetweenIterations() async throws { guard let vectorData = "{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -894,7 +1024,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector86TurnConformanceRunCancellationBetweenTools() async throws { + func testVector99TurnConformanceRunCancellationBetweenTools() async throws { guard let vectorData = "{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -904,7 +1034,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector87TurnConformanceRunContextNoTrimWhenFits() async throws { + func testVector100TurnConformanceRunContextNoTrimWhenFits() async throws { guard let vectorData = "{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -914,7 +1044,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector88TurnConformanceRunContextPreservesSystemMessages() async throws { + func testVector101TurnConformanceRunContextPreservesSystemMessages() async throws { guard let vectorData = "{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -924,7 +1054,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector89TurnConformanceRunContextTrimBasic() async throws { + func testVector102TurnConformanceRunContextTrimBasic() async throws { guard let vectorData = "{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -934,7 +1064,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector90TurnConformanceRunEmptyToolResult() async throws { + func testVector103TurnConformanceRunEmptyToolResult() async throws { guard let vectorData = "{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -944,7 +1074,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector91TurnConformanceRunEventsBasicToolLoop() async throws { + func testVector104TurnConformanceRunEventsBasicToolLoop() async throws { guard let vectorData = "{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -954,7 +1084,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector92TurnConformanceRunEventsErrorLogged() async throws { + func testVector105TurnConformanceRunEventsErrorLogged() async throws { guard let vectorData = "{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -964,7 +1094,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector93TurnConformanceRunEventsNoTools() async throws { + func testVector106TurnConformanceRunEventsNoTools() async throws { guard let vectorData = "{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -974,7 +1104,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector94TurnConformanceRunGuardrailAllPass() async throws { + func testVector107TurnConformanceRunGuardrailAllPass() async throws { guard let vectorData = "{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -984,7 +1114,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector95TurnConformanceRunGuardrailInputDeny() async throws { + func testVector108TurnConformanceRunGuardrailInputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -994,7 +1124,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector96TurnConformanceRunGuardrailOutputDeny() async throws { + func testVector109TurnConformanceRunGuardrailOutputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1004,7 +1134,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector97TurnConformanceRunGuardrailToolDeny() async throws { + func testVector110TurnConformanceRunGuardrailToolDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1014,7 +1144,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector98TurnConformanceRunMaxIterationsExceeded() async throws { + func testVector111TurnConformanceRunMaxIterationsExceeded() async throws { guard let vectorData = "{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1024,7 +1154,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector99TurnConformanceRunMultiTurnToolCalls() async throws { + func testVector112TurnConformanceRunMultiTurnToolCalls() async throws { guard let vectorData = "{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1034,7 +1164,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector100TurnConformanceRunMultipleToolCallsSingleTurn() async throws { + func testVector113TurnConformanceRunMultipleToolCallsSingleTurn() async throws { guard let vectorData = "{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1044,7 +1174,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector101TurnConformanceRunNoToolCalls() async throws { + func testVector114TurnConformanceRunNoToolCalls() async throws { guard let vectorData = "{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1054,7 +1184,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector102TurnConformanceRunParallelToolsBasic() async throws { + func testVector115TurnConformanceRunParallelToolsBasic() async throws { guard let vectorData = "{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1064,7 +1194,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector103TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { + func testVector116TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1074,7 +1204,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector104TurnConformanceRunParallelToolsWithToolError() async throws { + func testVector117TurnConformanceRunParallelToolsWithToolError() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\u{a7}13.6 Parallel Tools + tool error \u{2014} 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1084,7 +1214,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector105TurnConformanceRunSingleToolCall() async throws { + func testVector118TurnConformanceRunSingleToolCall() async throws { guard let vectorData = "{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1094,7 +1224,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector106TurnConformanceRunSteeringInjectMessage() async throws { + func testVector119TurnConformanceRunSteeringInjectMessage() async throws { guard let vectorData = "{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1104,7 +1234,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector107TurnConformanceRunSteeringMultipleMessages() async throws { + func testVector120TurnConformanceRunSteeringMultipleMessages() async throws { guard let vectorData = "{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1114,7 +1244,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector108TurnConformanceRunToolNotRegisteredError() async throws { + func testVector121TurnConformanceRunToolNotRegisteredError() async throws { guard let vectorData = "{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1124,7 +1254,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector109TurnConformanceRunToolResultMessageFormat() async throws { + func testVector122TurnConformanceRunToolResultMessageFormat() async throws { guard let vectorData = "{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1134,7 +1264,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector110TurnConformanceRunTurnCancelBeforeContext() async throws { + func testVector123TurnConformanceRunTurnCancelBeforeContext() async throws { guard let vectorData = "{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1144,7 +1274,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector111TurnConformanceRunTurnDelegatedProviderState() async throws { + func testVector124TurnConformanceRunTurnDelegatedProviderState() async throws { guard let vectorData = "{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1154,7 +1284,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector112TurnConformanceRunTurnFinalOutput() async throws { + func testVector125TurnConformanceRunTurnFinalOutput() async throws { guard let vectorData = "{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1164,7 +1294,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector113TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { + func testVector126TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { guard let vectorData = "{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1174,7 +1304,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector114TurnConformanceRunTurnOrderedToolRound() async throws { + func testVector127TurnConformanceRunTurnOrderedToolRound() async throws { guard let vectorData = "{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1184,7 +1314,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector115TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { + func testVector128TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { guard let vectorData = "{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1194,7 +1324,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector116WireConformanceToRequestAnthropicImageFormat() async throws { + func testVector129WireConformanceToRequestAnthropicImageFormat() async throws { guard let vectorData = "{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1204,7 +1334,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector117WireConformanceToRequestAnthropicMaxTokensRequired() async throws { + func testVector130WireConformanceToRequestAnthropicMaxTokensRequired() async throws { guard let vectorData = "{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1214,7 +1344,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector118WireConformanceToRequestAnthropicOptions() async throws { + func testVector131WireConformanceToRequestAnthropicOptions() async throws { guard let vectorData = "{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1224,7 +1354,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector119WireConformanceToRequestAnthropicSystemSeparate() async throws { + func testVector132WireConformanceToRequestAnthropicSystemSeparate() async throws { guard let vectorData = "{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1234,7 +1364,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector120WireConformanceToRequestAnthropicToolWire() async throws { + func testVector133WireConformanceToRequestAnthropicToolWire() async throws { guard let vectorData = "{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1244,7 +1374,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector121WireConformanceToRequestChatAudioMp3() async throws { + func testVector134WireConformanceToRequestChatAudioMp3() async throws { guard let vectorData = "{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1254,7 +1384,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector122WireConformanceToRequestChatAudioPart() async throws { + func testVector135WireConformanceToRequestChatAudioPart() async throws { guard let vectorData = "{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1264,7 +1394,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector123WireConformanceToRequestChatImageBase64() async throws { + func testVector136WireConformanceToRequestChatImageBase64() async throws { guard let vectorData = "{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1274,7 +1404,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector124WireConformanceToRequestChatImagePart() async throws { + func testVector137WireConformanceToRequestChatImagePart() async throws { guard let vectorData = "{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1284,7 +1414,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector125WireConformanceToRequestChatMultipartContent() async throws { + func testVector138WireConformanceToRequestChatMultipartContent() async throws { guard let vectorData = "{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1294,7 +1424,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector126WireConformanceToRequestChatSimple() async throws { + func testVector139WireConformanceToRequestChatSimple() async throws { guard let vectorData = "{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1304,7 +1434,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector127WireConformanceToRequestChatSingleTextOptimized() async throws { + func testVector140WireConformanceToRequestChatSingleTextOptimized() async throws { guard let vectorData = "{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1314,7 +1444,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector128WireConformanceToRequestChatWithOptions() async throws { + func testVector141WireConformanceToRequestChatWithOptions() async throws { guard let vectorData = "{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1324,7 +1454,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector129WireConformanceToRequestEmbeddingWire() async throws { + func testVector142WireConformanceToRequestEmbeddingWire() async throws { guard let vectorData = "{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1334,7 +1464,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector130WireConformanceToRequestImageWire() async throws { + func testVector143WireConformanceToRequestImageWire() async throws { guard let vectorData = "{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1344,7 +1474,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector131WireConformanceToRequestKindToJsonTypeMapping() async throws { + func testVector144WireConformanceToRequestKindToJsonTypeMapping() async throws { guard let vectorData = "{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1354,7 +1484,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector132WireConformanceToRequestOptionsAdditionalProperties() async throws { + func testVector145WireConformanceToRequestOptionsAdditionalProperties() async throws { guard let vectorData = "{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1364,7 +1494,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector133WireConformanceToRequestOptionsMaxCompletionTokens() async throws { + func testVector146WireConformanceToRequestOptionsMaxCompletionTokens() async throws { guard let vectorData = "{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1374,7 +1504,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector134WireConformanceToRequestOptionsStopSequences() async throws { + func testVector147WireConformanceToRequestOptionsStopSequences() async throws { guard let vectorData = "{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1384,7 +1514,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector135WireConformanceToRequestResponsesSimple() async throws { + func testVector148WireConformanceToRequestResponsesSimple() async throws { guard let vectorData = "{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1394,7 +1524,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector136WireConformanceToRequestResponsesStructuredOutput() async throws { + func testVector149WireConformanceToRequestResponsesStructuredOutput() async throws { guard let vectorData = "{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1404,7 +1534,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector137WireConformanceToRequestResponsesWithTools() async throws { + func testVector150WireConformanceToRequestResponsesWithTools() async throws { guard let vectorData = "{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1414,7 +1544,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector138WireConformanceToRequestStructuredOutput() async throws { + func testVector151WireConformanceToRequestStructuredOutput() async throws { guard let vectorData = "{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1424,7 +1554,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector139WireConformanceToRequestStructuredOutputNestedOptional() async throws { + func testVector152WireConformanceToRequestStructuredOutputNestedOptional() async throws { guard let vectorData = "{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1434,7 +1564,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector140WireConformanceToRequestToolsBindingsStripped() async throws { + func testVector153WireConformanceToRequestToolsBindingsStripped() async throws { guard let vectorData = "{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1444,7 +1574,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector141WireConformanceToRequestToolsFunctionWire() async throws { + func testVector154WireConformanceToRequestToolsFunctionWire() async throws { guard let vectorData = "{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1454,7 +1584,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector142WireConformanceToRequestToolsNullWhenEmpty() async throws { + func testVector155WireConformanceToRequestToolsNullWhenEmpty() async throws { guard let vectorData = "{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1464,7 +1594,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector143WireConformanceToRequestToolsStrictMode() async throws { + func testVector156WireConformanceToRequestToolsStrictMode() async throws { guard let vectorData = "{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { diff --git a/runtime/typescript/packages/core/src/model/vector-conformance.ts b/runtime/typescript/packages/core/src/model/vector-conformance.ts index ce3d820ae..a84e4188c 100644 --- a/runtime/typescript/packages/core/src/model/vector-conformance.ts +++ b/runtime/typescript/packages/core/src/model/vector-conformance.ts @@ -676,6 +676,20 @@ export async function runLoadConformanceConformance( "trusted_endpoint_builtin_allowed misrouted", ); } + // vector: trusted_endpoint_default_port_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://API.OpenAI.com:443/v1"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_default_port_allowed misrouted", + ); + } // vector: trusted_endpoint_env_endpoint_allowed { const sourceJson = "null"; @@ -746,6 +760,20 @@ export async function runLoadConformanceConformance( "trusted_endpoint_reference_skipped misrouted", ); } + // vector: trusted_endpoint_userinfo_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://user@api.openai.com/v1"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_userinfo_allowed misrouted", + ); + } // vector: untrusted_endpoint_ambient_rejected { const sourceJson = "null"; @@ -821,6 +849,36 @@ export async function runLoadConformanceConformance( "untrusted_endpoint_file_connection_rejected: expected an error", ); } + // vector: untrusted_endpoint_fragment_before_at_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_fragment_before_at_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_ipv6_literal_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_ipv6_literal_rejected: expected an error", + ); + } // vector: untrusted_endpoint_key_with_client_secret_rejected { const sourceJson = "null"; @@ -866,6 +924,81 @@ export async function runLoadConformanceConformance( "untrusted_endpoint_no_key_rejected: expected an error", ); } + // vector: untrusted_endpoint_percent_encoded_host_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_percent_encoded_host_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_port_mismatch_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_port_mismatch_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_query_before_at_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_query_before_at_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_scheme_mismatch_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_scheme_mismatch_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_suffix_lookalike_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_suffix_lookalike_rejected: expected an error", + ); + } // vector: untrusted_endpoint_suffix_spoof_rejected { const sourceJson = "null"; @@ -881,6 +1014,51 @@ export async function runLoadConformanceConformance( "untrusted_endpoint_suffix_spoof_rejected: expected an error", ); } + // vector: untrusted_endpoint_trailing_dot_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_trailing_dot_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_unicode_homoglyph_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_unicode_homoglyph_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_userinfo_lookalike_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_userinfo_lookalike_rejected: expected an error", + ); + } // vector: untrusted_endpoint_wildcard_apex_rejected { const sourceJson = "null"; @@ -896,4 +1074,19 @@ export async function runLoadConformanceConformance( "untrusted_endpoint_wildcard_apex_rejected: expected an error", ); } + // vector: untrusted_endpoint_wildcard_boundary_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_wildcard_boundary_rejected: expected an error", + ); + } } diff --git a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts index 4799b1531..2797b945a 100644 --- a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts +++ b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts @@ -333,6 +333,12 @@ describe("callable vector conformance", () => { ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:trusted_endpoint_default_port_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_default_port_allowed","description":"An explicit default port normalizes to the built-in origin","stage":"load","input":{"frontmatter":{"name":"trusted-default-port-allowed","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://API.OpenAI.com:443/v1"}}}},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://API.OpenAI.com:443/v1"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("LoadConformance.load:trusted_endpoint_env_endpoint_allowed", async () => { const vector = JSON.parse( '{"name":"trusted_endpoint_env_endpoint_allowed","description":"An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials","stage":"load","input":{"frontmatter":{"name":"trusted-env-endpoint","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"${env:VECTOR_FOUNDRY_ENDPOINT}"}}},"env":{"VECTOR_FOUNDRY_ENDPOINT":"https://host-chosen.example.com"}},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://host-chosen.example.com"}}},"operation":"load"}', @@ -363,6 +369,12 @@ describe("callable vector conformance", () => { ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:trusted_endpoint_userinfo_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_userinfo_allowed","description":"Userinfo is ignored; the destination host api.openai.com is trusted","stage":"load","input":{"frontmatter":{"name":"trusted-userinfo-allowed","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://user@api.openai.com/v1"}}}},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://user@api.openai.com/v1"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("LoadConformance.load:untrusted_endpoint_ambient_rejected", async () => { const vector = JSON.parse( '{"name":"untrusted_endpoint_ambient_rejected","description":"A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)","stage":"load","input":{"frontmatter":{"name":"untrusted-ambient","model":{"id":"gpt-4","provider":"foundry","connection":{"kind":"foundry","endpoint":"https://attacker.example.com/api/projects/p"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', @@ -393,6 +405,18 @@ describe("callable vector conformance", () => { ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:untrusted_endpoint_fragment_before_at_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_fragment_before_at_rejected","description":"A \'#\' ends the authority, so the host is evil.example.com","stage":"load","input":{"frontmatter":{"name":"untrusted-fragment-before-at","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://evil.example.com#@api.openai.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_ipv6_literal_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_ipv6_literal_rejected","description":"An IPv6 literal is not a trusted origin","stage":"load","input":{"frontmatter":{"name":"untrusted-ipv6-literal","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://[::1]:8080"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", async () => { const vector = JSON.parse( '{"name":"untrusted_endpoint_key_with_client_secret_rejected","description":"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-key-client-secret","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"https://attacker.example.com","clientSecret":"junk"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', @@ -411,18 +435,72 @@ describe("callable vector conformance", () => { ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:untrusted_endpoint_percent_encoded_host_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_percent_encoded_host_rejected","description":"Hosts are compared without percent-decoding, so an encoded host never matches","stage":"load","input":{"frontmatter":{"name":"untrusted-percent-encoded-host","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com%2eevil.example.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_port_mismatch_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_port_mismatch_rejected","description":"A non-default port is a different origin","stage":"load","input":{"frontmatter":{"name":"untrusted-port-mismatch","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com:8443"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_query_before_at_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_query_before_at_rejected","description":"A \'?\' ends the authority, so the host is evil.example.com","stage":"load","input":{"frontmatter":{"name":"untrusted-query-before-at","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://evil.example.com?@api.openai.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_scheme_mismatch_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_scheme_mismatch_rejected","description":"Built-in origins are https; http on the same host is a different origin","stage":"load","input":{"frontmatter":{"name":"untrusted-scheme-mismatch","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"http://api.openai.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_suffix_lookalike_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_suffix_lookalike_rejected","description":"A trusted host as a label prefix of another domain is not trusted","stage":"load","input":{"frontmatter":{"name":"untrusted-suffix-lookalike","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com.evil.example.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", async () => { const vector = JSON.parse( '{"name":"untrusted_endpoint_suffix_spoof_rejected","description":"Origin matching is exact: a host that merely starts with a trusted host is not trusted","stage":"load","input":{"frontmatter":{"name":"untrusted-spoof","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://good.example.com.attacker.example.com"}}},"trustedEndpoints":["https://good.example.com"]},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:untrusted_endpoint_trailing_dot_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_trailing_dot_rejected","description":"Hosts are compared exactly; a trailing-dot host does not match","stage":"load","input":{"frontmatter":{"name":"untrusted-trailing-dot","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com."}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_unicode_homoglyph_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_unicode_homoglyph_rejected","description":"A non-ASCII lookalike host does not match","stage":"load","input":{"frontmatter":{"name":"untrusted-unicode-homoglyph","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://аpi.openai.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_userinfo_lookalike_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_userinfo_lookalike_rejected","description":"A trusted host in userinfo is not the destination host","stage":"load","input":{"frontmatter":{"name":"untrusted-userinfo-lookalike","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com@evil.example.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", async () => { const vector = JSON.parse( '{"name":"untrusted_endpoint_wildcard_apex_rejected","description":"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme","stage":"load","input":{"frontmatter":{"name":"untrusted-apex","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"http://myres.openai.azure.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:untrusted_endpoint_wildcard_boundary_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_wildcard_boundary_rejected","description":"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com","stage":"load","input":{"frontmatter":{"name":"untrusted-wildcard-boundary","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://evilservices.ai.azure.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("MemoryConformance.operate:clear_by_category_and_all", async () => { const vector = JSON.parse( '{"name":"clear_by_category_and_all","description":"Clear removes only the requested category, or all memories when no category is supplied.","stage":"memory","input":{"operation":"clear","store":{"entries":[{"content":"fact","category":"core"},{"content":"summary","category":"archival"},{"content":"another summary","category":"archival"},{"content":"insight","category":"insight"}]},"category":"archival"},"expected":{"removed":2,"store":{"entries":[{"content":"fact","category":"core"},{"content":"insight","category":"insight"}]}},"operation":"operate"}', diff --git a/schema/model/conformance/vectors/load.tsp b/schema/model/conformance/vectors/load.tsp index b6d8a01db..396b5ec11 100644 --- a/schema/model/conformance/vectors/load.tsp +++ b/schema/model/conformance/vectors/load.tsp @@ -397,6 +397,276 @@ const LoadVectors = """ "kind": "untrusted_endpoint" } }, + { + "name": "untrusted_endpoint_fragment_before_at_rejected", + "description": "A '#' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-fragment-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com#@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_query_before_at_rejected", + "description": "A '?' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-query-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com?@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_userinfo_lookalike_rejected", + "description": "A trusted host in userinfo is not the destination host", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-userinfo-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com@evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_suffix_lookalike_rejected", + "description": "A trusted host as a label prefix of another domain is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-suffix-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com.evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_wildcard_boundary_rejected", + "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-wildcard-boundary", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evilservices.ai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_scheme_mismatch_rejected", + "description": "Built-in origins are https; http on the same host is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-scheme-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_port_mismatch_rejected", + "description": "A non-default port is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-port-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com:8443" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_ipv6_literal_rejected", + "description": "An IPv6 literal is not a trusted origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ipv6-literal", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://[::1]:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_percent_encoded_host_rejected", + "description": "Hosts are compared without percent-decoding, so an encoded host never matches", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-percent-encoded-host", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com%2eevil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_trailing_dot_rejected", + "description": "Hosts are compared exactly; a trailing-dot host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-trailing-dot", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com." + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_unicode_homoglyph_rejected", + "description": "A non-ASCII lookalike host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-unicode-homoglyph", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://аpi.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "trusted_endpoint_userinfo_allowed", + "description": "Userinfo is ignored; the destination host api.openai.com is trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-userinfo-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + { + "name": "trusted_endpoint_default_port_allowed", + "description": "An explicit default port normalizes to the built-in origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-default-port-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, { "name": "untrusted_endpoint_env_default_rejected", "description": "An \${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", diff --git a/schema/tsp-output/.typra-generated/vectors.json b/schema/tsp-output/.typra-generated/vectors.json index c7ad8c695..15ac46f74 100644 --- a/schema/tsp-output/.typra-generated/vectors.json +++ b/schema/tsp-output/.typra-generated/vectors.json @@ -1894,6 +1894,43 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_default_port_allowed", + "description": "An explicit default port normalizes to the built-in origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-default-port-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + }, + "operation": "load" + } + }, { "contract": "LoadConformance", "namespace": "Prompty.Conformance.Seams", @@ -2089,6 +2126,43 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_userinfo_allowed", + "description": "Userinfo is ignored; the destination host api.openai.com is trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-userinfo-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + }, + "operation": "load" + } + }, { "contract": "LoadConformance", "namespace": "Prompty.Conformance.Seams", @@ -2258,6 +2332,70 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_fragment_before_at_rejected", + "description": "A '#' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-fragment-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com#@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_ipv6_literal_rejected", + "description": "An IPv6 literal is not a trusted origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ipv6-literal", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://[::1]:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, { "contract": "LoadConformance", "namespace": "Prompty.Conformance.Seams", @@ -2356,6 +2494,166 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_percent_encoded_host_rejected", + "description": "Hosts are compared without percent-decoding, so an encoded host never matches", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-percent-encoded-host", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com%2eevil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_port_mismatch_rejected", + "description": "A non-default port is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-port-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com:8443" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_query_before_at_rejected", + "description": "A '?' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-query-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com?@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_scheme_mismatch_rejected", + "description": "Built-in origins are https; http on the same host is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-scheme-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_suffix_lookalike_rejected", + "description": "A trusted host as a label prefix of another domain is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-suffix-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com.evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, { "contract": "LoadConformance", "namespace": "Prompty.Conformance.Seams", @@ -2391,6 +2689,102 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_trailing_dot_rejected", + "description": "Hosts are compared exactly; a trailing-dot host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-trailing-dot", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com." + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_unicode_homoglyph_rejected", + "description": "A non-ASCII lookalike host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-unicode-homoglyph", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://аpi.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_userinfo_lookalike_rejected", + "description": "A trusted host in userinfo is not the destination host", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-userinfo-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com@evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, { "contract": "LoadConformance", "namespace": "Prompty.Conformance.Seams", @@ -2423,6 +2817,38 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_wildcard_boundary_rejected", + "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-wildcard-boundary", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evilservices.ai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, { "contract": "MemoryConformance", "namespace": "Prompty.Conformance.Seams", diff --git a/spec/spec.md b/spec/spec.md index fb86beb8a..98e3ecedb 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -324,7 +324,9 @@ Built-in: `https://api.openai.com`, `https://api.anthropic.com`, **Matching**: compare origins only (scheme + host + port). Scheme and host are case-insensitive; default ports (`http`:80, `https`:443) are normalized; path, query, -and userinfo are ignored. An entry `scheme://*.domain[:port]` matches any strict +and userinfo are ignored. The authority ends at the first `/`, `?`, or `#`. Hosts are +otherwise compared as written: no percent-decoding, IDNA mapping, or trailing-dot +stripping, so such hosts never match a built-in entry. An entry `scheme://*.domain[:port]` matches any strict subdomain of `domain` (not `domain` itself). The entry `*` trusts every endpoint. A destination that does not parse as an absolute `http`/`https` URL, or that contains a backslash (`\`), is untrusted. From 75fb665c422fdb8507fb55cbcb5fc37ef931911e Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 18:08:47 -0700 Subject: [PATCH 12/15] fix(swift): read live connection apiKey in OpenAI and Anthropic configs Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../PromptyAnthropic/AnthropicConfig.swift | 5 +- .../Sources/PromptyOpenAI/OpenAIConfig.swift | 5 +- .../PromptyTests/ConnectionKeyTests.swift | 49 +++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 runtime/swift/prompty/Tests/PromptyTests/ConnectionKeyTests.swift diff --git a/runtime/swift/prompty/Sources/PromptyAnthropic/AnthropicConfig.swift b/runtime/swift/prompty/Sources/PromptyAnthropic/AnthropicConfig.swift index 9e651fb53..395102ebe 100644 --- a/runtime/swift/prompty/Sources/PromptyAnthropic/AnthropicConfig.swift +++ b/runtime/swift/prompty/Sources/PromptyAnthropic/AnthropicConfig.swift @@ -27,7 +27,10 @@ struct AnthropicConfig { if let connection { let fields = (try? connection.save()) ?? [:] if let value = fields["endpoint"] as? String, !value.isEmpty { endpoint = value } - if let value = fields["apiKey"] as? String, !value.isEmpty { apiKey = value } + // apiKey is withheld from save(); read the live value from the typed connection. + if case .apiKeyConnection(let keyed) = connection, !keyed.apiKey.isEmpty { + apiKey = keyed.apiKey + } } if endpoint.isEmpty { diff --git a/runtime/swift/prompty/Sources/PromptyOpenAI/OpenAIConfig.swift b/runtime/swift/prompty/Sources/PromptyOpenAI/OpenAIConfig.swift index 691075061..777663715 100644 --- a/runtime/swift/prompty/Sources/PromptyOpenAI/OpenAIConfig.swift +++ b/runtime/swift/prompty/Sources/PromptyOpenAI/OpenAIConfig.swift @@ -28,7 +28,10 @@ struct OpenAIConfig { if let connection { let fields = connectionFields(connection) if let value = fields["endpoint"] as? String, !value.isEmpty { endpoint = value } - if let value = fields["apiKey"] as? String, !value.isEmpty { apiKey = value } + // apiKey is withheld from save(); read the live value from the typed connection. + if case .apiKeyConnection(let keyed) = connection, !keyed.apiKey.isEmpty { + apiKey = keyed.apiKey + } if let value = fields["headers"] as? [String: Any] { for (key, header) in value { headers[key] = JSONSupport.stringify(header) } } diff --git a/runtime/swift/prompty/Tests/PromptyTests/ConnectionKeyTests.swift b/runtime/swift/prompty/Tests/PromptyTests/ConnectionKeyTests.swift new file mode 100644 index 000000000..fbf28932a --- /dev/null +++ b/runtime/swift/prompty/Tests/PromptyTests/ConnectionKeyTests.swift @@ -0,0 +1,49 @@ +import Foundation +import PromptyModel +import XCTest + +@testable import Prompty +@testable import PromptyAnthropic +@testable import PromptyOpenAI + +/// A file-supplied `apiKey` must be the key sent, never the ambient env key +/// (`apiKey` is withheld from `save()`, so configs must read the live value). +final class ConnectionKeyTests: XCTestCase { + + private func agent(provider: String) throws -> Agent { + try Agent.load([ + "kind": "prompt", + "name": "connection-key", + "model": [ + "id": "m", + "provider": provider, + "connection": [ + "kind": "key", + "endpoint": "https://custom.example.com", + "apiKey": "file-key", + ], + ], + ]) + } + + private var savedEnv: [String: String?] = [:] + + override func setUp() { + for key in ["ANTHROPIC_API_KEY", "OPENAI_API_KEY"] { + savedEnv[key] = ProcessInfo.processInfo.environment[key] + Env.set(key, "ambient-key") + } + } + + override func tearDown() { + for (key, value) in savedEnv { Env.set(key, value) } + } + + func testAnthropicUsesFileKey() throws { + XCTAssertEqual(try AnthropicConfig.resolve(agent(provider: "anthropic")).apiKey, "file-key") + } + + func testOpenAIUsesFileKey() throws { + XCTAssertEqual(try OpenAIConfig.resolve(agent(provider: "openai")).apiKey, "file-key") + } +} From ee47fbd79ccfd1a377fbf604750216605fb60391 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 18:08:48 -0700 Subject: [PATCH 13/15] fix(spec): treat oauth connections as ambient in endpoint trust Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Model/VectorConformanceTests.cs | 216 +++++----- runtime/csharp/Prompty.Core/EndpointTrust.cs | 3 +- .../Prompty.Core/Model/VectorConformance.cs | 14 + runtime/go/prompty/model/endpoint_trust.go | 2 +- .../prompty/model/vector_conformance_test.go | 215 +++++----- .../vectorconformance/vector_conformance.go | 11 + runtime/java/README.md | 3 +- .../com/microsoft/prompty/EndpointTrust.java | 2 +- .../prompty/model/VectorConformance.java | 14 + .../prompty/model/VectorConformanceTests.java | 372 +++++++++--------- runtime/python/prompty/README.md | 2 +- .../prompty/prompty/core/endpoint_trust.py | 2 +- .../prompty/model/vector_conformance.py | 10 + .../tests/model/test_vector_conformance.py | 214 +++++----- runtime/rust/prompty/README.md | 2 +- .../rust/prompty/src/loader/endpoint_trust.rs | 2 +- .../prompty/src/model/vector_conformance.rs | 14 + .../tests/model/vector_conformance_test.rs | 189 ++++----- .../Sources/PromptyModel/EndpointTrust.swift | 2 +- .../PromptyModel/vector_conformance.swift | 17 + .../VectorConformanceTests.swift | 192 ++++----- .../packages/core/src/core/endpoint-trust.ts | 3 +- .../core/src/model/vector-conformance.ts | 15 + .../tests/model/vector-conformance.test.ts | 6 + schema/model/conformance/vectors/load.tsp | 24 ++ .../tsp-output/.typra-generated/vectors.json | 36 ++ spec/spec.md | 10 +- .../docs/specification/file-format.mdx | 3 +- 28 files changed, 940 insertions(+), 655 deletions(-) diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs index 53727818a..aea8e4ab9 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs @@ -2398,7 +2398,41 @@ public async Task Vector65LoadConformanceLoadUntrustedEndpointNoKeyRejected() } [Fact] - public async Task Vector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() + public async Task Vector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_oauth_literal_secret_rejected", + "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-oauth-literal", + "model": { + "id": "gpt-4", + "provider": "anthropic", + "connection": { + "kind": "oauth", + "endpoint": "https://attacker.example.com", + "clientId": "decoy", + "clientSecret": "decoy", + "tokenUrl": "https://login.microsoftonline.com/t/oauth2/v2.0/token" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() { string vectorJson = """ { @@ -2428,7 +2462,7 @@ public async Task Vector66LoadConformanceLoadUntrustedEndpointPercentEncodedHost } [Fact] - public async Task Vector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected() + public async Task Vector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected() { string vectorJson = """ { @@ -2458,7 +2492,7 @@ public async Task Vector67LoadConformanceLoadUntrustedEndpointPortMismatchReject } [Fact] - public async Task Vector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() + public async Task Vector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() { string vectorJson = """ { @@ -2488,7 +2522,7 @@ public async Task Vector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejec } [Fact] - public async Task Vector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() + public async Task Vector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() { string vectorJson = """ { @@ -2518,7 +2552,7 @@ public async Task Vector69LoadConformanceLoadUntrustedEndpointSchemeMismatchReje } [Fact] - public async Task Vector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() + public async Task Vector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() { string vectorJson = """ { @@ -2548,7 +2582,7 @@ public async Task Vector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRej } [Fact] - public async Task Vector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() + public async Task Vector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() { string vectorJson = """ { @@ -2581,7 +2615,7 @@ public async Task Vector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejecte } [Fact] - public async Task Vector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected() + public async Task Vector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected() { string vectorJson = """ { @@ -2611,7 +2645,7 @@ public async Task Vector72LoadConformanceLoadUntrustedEndpointTrailingDotRejecte } [Fact] - public async Task Vector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() + public async Task Vector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() { string vectorJson = """ { @@ -2641,7 +2675,7 @@ public async Task Vector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRe } [Fact] - public async Task Vector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() + public async Task Vector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() { string vectorJson = """ { @@ -2671,7 +2705,7 @@ public async Task Vector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeR } [Fact] - public async Task Vector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected() + public async Task Vector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected() { string vectorJson = """ { @@ -2701,7 +2735,7 @@ public async Task Vector75LoadConformanceLoadUntrustedEndpointWildcardApexReject } [Fact] - public async Task Vector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() + public async Task Vector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() { string vectorJson = """ { @@ -2731,7 +2765,7 @@ public async Task Vector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRe } [Fact] - public async Task Vector77MemoryConformanceOperateClearByCategoryAndAll() + public async Task Vector78MemoryConformanceOperateClearByCategoryAndAll() { string vectorJson = """ { @@ -2785,7 +2819,7 @@ public async Task Vector77MemoryConformanceOperateClearByCategoryAndAll() } [Fact] - public async Task Vector78MemoryConformanceOperateFormatCoreAndRecallResults() + public async Task Vector79MemoryConformanceOperateFormatCoreAndRecallResults() { string vectorJson = """ { @@ -2827,7 +2861,7 @@ public async Task Vector78MemoryConformanceOperateFormatCoreAndRecallResults() } [Fact] - public async Task Vector79MemoryConformanceOperateRecallEmptyQueryAndLimit() + public async Task Vector80MemoryConformanceOperateRecallEmptyQueryAndLimit() { string vectorJson = """ { @@ -2882,7 +2916,7 @@ public async Task Vector79MemoryConformanceOperateRecallEmptyQueryAndLimit() } [Fact] - public async Task Vector80MemoryConformanceOperateRecallRanksByWeightedScore() + public async Task Vector81MemoryConformanceOperateRecallRanksByWeightedScore() { string vectorJson = """ { @@ -2941,7 +2975,7 @@ public async Task Vector80MemoryConformanceOperateRecallRanksByWeightedScore() } [Fact] - public async Task Vector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() + public async Task Vector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() { string vectorJson = """ { @@ -2999,7 +3033,7 @@ public async Task Vector81MemoryConformanceOperateRecallScoreTiesPreserveInserti } [Fact] - public async Task Vector82MemoryConformanceOperateRecallTagsAndCoreBoost() + public async Task Vector83MemoryConformanceOperateRecallTagsAndCoreBoost() { string vectorJson = """ { @@ -3060,7 +3094,7 @@ public async Task Vector82MemoryConformanceOperateRecallTagsAndCoreBoost() } [Fact] - public async Task Vector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() + public async Task Vector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() { string vectorJson = """ { @@ -3121,7 +3155,7 @@ public async Task Vector83MemoryConformanceOperateRecallTokenizationPunctuationU } [Fact] - public async Task Vector84MemoryConformanceOperateRememberDedupsCoreByTags() + public async Task Vector85MemoryConformanceOperateRememberDedupsCoreByTags() { string vectorJson = """ { @@ -3185,7 +3219,7 @@ public async Task Vector84MemoryConformanceOperateRememberDedupsCoreByTags() } [Fact] - public async Task Vector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() + public async Task Vector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() { string vectorJson = """ { @@ -3241,7 +3275,7 @@ public async Task Vector85MemoryConformanceOperateRememberDedupsCoreMissingAndEm } [Fact] - public async Task Vector86MemoryConformanceOperateRememberEvictsArchivalFirst() + public async Task Vector87MemoryConformanceOperateRememberEvictsArchivalFirst() { string vectorJson = """ { @@ -3302,7 +3336,7 @@ public async Task Vector86MemoryConformanceOperateRememberEvictsArchivalFirst() } [Fact] - public async Task Vector87MemoryConformanceOperateSnapshotPortRoundTrip() + public async Task Vector88MemoryConformanceOperateSnapshotPortRoundTrip() { string vectorJson = """ { @@ -3342,7 +3376,7 @@ public async Task Vector87MemoryConformanceOperateSnapshotPortRoundTrip() } [Fact] - public async Task Vector88MemoryConformanceOperateUpdateIndexOutOfRangeError() + public async Task Vector89MemoryConformanceOperateUpdateIndexOutOfRangeError() { string vectorJson = """ { @@ -3376,7 +3410,7 @@ public async Task Vector88MemoryConformanceOperateUpdateIndexOutOfRangeError() } [Fact] - public async Task Vector89TurnConformanceReplayMaxIterations() + public async Task Vector90TurnConformanceReplayMaxIterations() { string vectorJson = """ { @@ -3413,7 +3447,7 @@ public async Task Vector89TurnConformanceReplayMaxIterations() } [Fact] - public async Task Vector90TurnConformanceReplayNoTool() + public async Task Vector91TurnConformanceReplayNoTool() { string vectorJson = """ { @@ -3446,7 +3480,7 @@ public async Task Vector90TurnConformanceReplayNoTool() } [Fact] - public async Task Vector91TurnConformanceReplayPermissionDenied() + public async Task Vector92TurnConformanceReplayPermissionDenied() { string vectorJson = """ { @@ -3482,7 +3516,7 @@ public async Task Vector91TurnConformanceReplayPermissionDenied() } [Fact] - public async Task Vector92TurnConformanceReplayToolFailure() + public async Task Vector93TurnConformanceReplayToolFailure() { string vectorJson = """ { @@ -3520,7 +3554,7 @@ public async Task Vector92TurnConformanceReplayToolFailure() } [Fact] - public async Task Vector93TurnConformanceReplayToolSuccess() + public async Task Vector94TurnConformanceReplayToolSuccess() { string vectorJson = """ { @@ -3558,7 +3592,7 @@ public async Task Vector93TurnConformanceReplayToolSuccess() } [Fact] - public async Task Vector94TurnConformanceRunAssistantToolCallsMetadata() + public async Task Vector95TurnConformanceRunAssistantToolCallsMetadata() { string vectorJson = """ { @@ -3712,7 +3746,7 @@ public async Task Vector94TurnConformanceRunAssistantToolCallsMetadata() } [Fact] - public async Task Vector95TurnConformanceRunAsyncToolFunction() + public async Task Vector96TurnConformanceRunAsyncToolFunction() { string vectorJson = """ { @@ -3824,7 +3858,7 @@ public async Task Vector95TurnConformanceRunAsyncToolFunction() } [Fact] - public async Task Vector96TurnConformanceRunBindingsInjected() + public async Task Vector97TurnConformanceRunBindingsInjected() { string vectorJson = """ { @@ -3956,7 +3990,7 @@ public async Task Vector96TurnConformanceRunBindingsInjected() } [Fact] - public async Task Vector97TurnConformanceRunCancellationBeforeLlm() + public async Task Vector98TurnConformanceRunCancellationBeforeLlm() { string vectorJson = """ { @@ -4019,7 +4053,7 @@ public async Task Vector97TurnConformanceRunCancellationBeforeLlm() } [Fact] - public async Task Vector98TurnConformanceRunCancellationBetweenIterations() + public async Task Vector99TurnConformanceRunCancellationBetweenIterations() { string vectorJson = """ { @@ -4151,7 +4185,7 @@ public async Task Vector98TurnConformanceRunCancellationBetweenIterations() } [Fact] - public async Task Vector99TurnConformanceRunCancellationBetweenTools() + public async Task Vector100TurnConformanceRunCancellationBetweenTools() { string vectorJson = """ { @@ -4292,7 +4326,7 @@ public async Task Vector99TurnConformanceRunCancellationBetweenTools() } [Fact] - public async Task Vector100TurnConformanceRunContextNoTrimWhenFits() + public async Task Vector101TurnConformanceRunContextNoTrimWhenFits() { string vectorJson = """ { @@ -4365,7 +4399,7 @@ public async Task Vector100TurnConformanceRunContextNoTrimWhenFits() } [Fact] - public async Task Vector101TurnConformanceRunContextPreservesSystemMessages() + public async Task Vector102TurnConformanceRunContextPreservesSystemMessages() { string vectorJson = """ { @@ -4474,7 +4508,7 @@ public async Task Vector101TurnConformanceRunContextPreservesSystemMessages() } [Fact] - public async Task Vector102TurnConformanceRunContextTrimBasic() + public async Task Vector103TurnConformanceRunContextTrimBasic() { string vectorJson = """ { @@ -4634,7 +4668,7 @@ public async Task Vector102TurnConformanceRunContextTrimBasic() } [Fact] - public async Task Vector103TurnConformanceRunEmptyToolResult() + public async Task Vector104TurnConformanceRunEmptyToolResult() { string vectorJson = """ { @@ -4751,7 +4785,7 @@ public async Task Vector103TurnConformanceRunEmptyToolResult() } [Fact] - public async Task Vector104TurnConformanceRunEventsBasicToolLoop() + public async Task Vector105TurnConformanceRunEventsBasicToolLoop() { string vectorJson = """ { @@ -4899,7 +4933,7 @@ public async Task Vector104TurnConformanceRunEventsBasicToolLoop() } [Fact] - public async Task Vector105TurnConformanceRunEventsErrorLogged() + public async Task Vector106TurnConformanceRunEventsErrorLogged() { string vectorJson = """ { @@ -5027,7 +5061,7 @@ public async Task Vector105TurnConformanceRunEventsErrorLogged() } [Fact] - public async Task Vector106TurnConformanceRunEventsNoTools() + public async Task Vector107TurnConformanceRunEventsNoTools() { string vectorJson = """ { @@ -5107,7 +5141,7 @@ public async Task Vector106TurnConformanceRunEventsNoTools() } [Fact] - public async Task Vector107TurnConformanceRunGuardrailAllPass() + public async Task Vector108TurnConformanceRunGuardrailAllPass() { string vectorJson = """ { @@ -5232,7 +5266,7 @@ public async Task Vector107TurnConformanceRunGuardrailAllPass() } [Fact] - public async Task Vector108TurnConformanceRunGuardrailInputDeny() + public async Task Vector109TurnConformanceRunGuardrailInputDeny() { string vectorJson = """ { @@ -5290,7 +5324,7 @@ public async Task Vector108TurnConformanceRunGuardrailInputDeny() } [Fact] - public async Task Vector109TurnConformanceRunGuardrailOutputDeny() + public async Task Vector110TurnConformanceRunGuardrailOutputDeny() { string vectorJson = """ { @@ -5370,7 +5404,7 @@ public async Task Vector109TurnConformanceRunGuardrailOutputDeny() } [Fact] - public async Task Vector110TurnConformanceRunGuardrailToolDeny() + public async Task Vector111TurnConformanceRunGuardrailToolDeny() { string vectorJson = """ { @@ -5526,7 +5560,7 @@ public async Task Vector110TurnConformanceRunGuardrailToolDeny() } [Fact] - public async Task Vector111TurnConformanceRunMaxIterationsExceeded() + public async Task Vector112TurnConformanceRunMaxIterationsExceeded() { string vectorJson = """ { @@ -6038,7 +6072,7 @@ public async Task Vector111TurnConformanceRunMaxIterationsExceeded() } [Fact] - public async Task Vector112TurnConformanceRunMultiTurnToolCalls() + public async Task Vector113TurnConformanceRunMultiTurnToolCalls() { string vectorJson = """ { @@ -6217,7 +6251,7 @@ public async Task Vector112TurnConformanceRunMultiTurnToolCalls() } [Fact] - public async Task Vector113TurnConformanceRunMultipleToolCallsSingleTurn() + public async Task Vector114TurnConformanceRunMultipleToolCallsSingleTurn() { string vectorJson = """ { @@ -6400,7 +6434,7 @@ public async Task Vector113TurnConformanceRunMultipleToolCallsSingleTurn() } [Fact] - public async Task Vector114TurnConformanceRunNoToolCalls() + public async Task Vector115TurnConformanceRunNoToolCalls() { string vectorJson = """ { @@ -6470,7 +6504,7 @@ public async Task Vector114TurnConformanceRunNoToolCalls() } [Fact] - public async Task Vector115TurnConformanceRunParallelToolsBasic() + public async Task Vector116TurnConformanceRunParallelToolsBasic() { string vectorJson = """ { @@ -6645,7 +6679,7 @@ public async Task Vector115TurnConformanceRunParallelToolsBasic() } [Fact] - public async Task Vector116TurnConformanceRunParallelToolsWithGuardrailDeny() + public async Task Vector117TurnConformanceRunParallelToolsWithGuardrailDeny() { string vectorJson = """ { @@ -6838,7 +6872,7 @@ public async Task Vector116TurnConformanceRunParallelToolsWithGuardrailDeny() } [Fact] - public async Task Vector117TurnConformanceRunParallelToolsWithToolError() + public async Task Vector118TurnConformanceRunParallelToolsWithToolError() { string vectorJson = """ { @@ -7013,7 +7047,7 @@ public async Task Vector117TurnConformanceRunParallelToolsWithToolError() } [Fact] - public async Task Vector118TurnConformanceRunSingleToolCall() + public async Task Vector119TurnConformanceRunSingleToolCall() { string vectorJson = """ { @@ -7162,7 +7196,7 @@ public async Task Vector118TurnConformanceRunSingleToolCall() } [Fact] - public async Task Vector119TurnConformanceRunSteeringInjectMessage() + public async Task Vector120TurnConformanceRunSteeringInjectMessage() { string vectorJson = """ { @@ -7372,7 +7406,7 @@ public async Task Vector119TurnConformanceRunSteeringInjectMessage() } [Fact] - public async Task Vector120TurnConformanceRunSteeringMultipleMessages() + public async Task Vector121TurnConformanceRunSteeringMultipleMessages() { string vectorJson = """ { @@ -7544,7 +7578,7 @@ public async Task Vector120TurnConformanceRunSteeringMultipleMessages() } [Fact] - public async Task Vector121TurnConformanceRunToolNotRegisteredError() + public async Task Vector122TurnConformanceRunToolNotRegisteredError() { string vectorJson = """ { @@ -7630,7 +7664,7 @@ public async Task Vector121TurnConformanceRunToolNotRegisteredError() } [Fact] - public async Task Vector122TurnConformanceRunToolResultMessageFormat() + public async Task Vector123TurnConformanceRunToolResultMessageFormat() { string vectorJson = """ { @@ -7753,7 +7787,7 @@ public async Task Vector122TurnConformanceRunToolResultMessageFormat() } [Fact] - public async Task Vector123TurnConformanceRunTurnCancelBeforeContext() + public async Task Vector124TurnConformanceRunTurnCancelBeforeContext() { string vectorJson = """ { @@ -7787,7 +7821,7 @@ public async Task Vector123TurnConformanceRunTurnCancelBeforeContext() } [Fact] - public async Task Vector124TurnConformanceRunTurnDelegatedProviderState() + public async Task Vector125TurnConformanceRunTurnDelegatedProviderState() { string vectorJson = """ { @@ -7856,7 +7890,7 @@ public async Task Vector124TurnConformanceRunTurnDelegatedProviderState() } [Fact] - public async Task Vector125TurnConformanceRunTurnFinalOutput() + public async Task Vector126TurnConformanceRunTurnFinalOutput() { string vectorJson = """ { @@ -7903,7 +7937,7 @@ public async Task Vector125TurnConformanceRunTurnFinalOutput() } [Fact] - public async Task Vector126TurnConformanceRunTurnMemoryContextIsModelVisible() + public async Task Vector127TurnConformanceRunTurnMemoryContextIsModelVisible() { string vectorJson = """ { @@ -7967,7 +8001,7 @@ public async Task Vector126TurnConformanceRunTurnMemoryContextIsModelVisible() } [Fact] - public async Task Vector127TurnConformanceRunTurnOrderedToolRound() + public async Task Vector128TurnConformanceRunTurnOrderedToolRound() { string vectorJson = """ { @@ -8060,7 +8094,7 @@ public async Task Vector127TurnConformanceRunTurnOrderedToolRound() } [Fact] - public async Task Vector128TurnConformanceRunTurnPermissionDenialIsModelVisible() + public async Task Vector129TurnConformanceRunTurnPermissionDenialIsModelVisible() { string vectorJson = """ { @@ -8113,7 +8147,7 @@ public async Task Vector128TurnConformanceRunTurnPermissionDenialIsModelVisible( } [Fact] - public async Task Vector129WireConformanceToRequestAnthropicImageFormat() + public async Task Vector130WireConformanceToRequestAnthropicImageFormat() { string vectorJson = """ { @@ -8185,7 +8219,7 @@ public async Task Vector129WireConformanceToRequestAnthropicImageFormat() } [Fact] - public async Task Vector130WireConformanceToRequestAnthropicMaxTokensRequired() + public async Task Vector131WireConformanceToRequestAnthropicMaxTokensRequired() { string vectorJson = """ { @@ -8242,7 +8276,7 @@ public async Task Vector130WireConformanceToRequestAnthropicMaxTokensRequired() } [Fact] - public async Task Vector131WireConformanceToRequestAnthropicOptions() + public async Task Vector132WireConformanceToRequestAnthropicOptions() { string vectorJson = """ { @@ -8313,7 +8347,7 @@ public async Task Vector131WireConformanceToRequestAnthropicOptions() } [Fact] - public async Task Vector132WireConformanceToRequestAnthropicSystemSeparate() + public async Task Vector133WireConformanceToRequestAnthropicSystemSeparate() { string vectorJson = """ { @@ -8382,7 +8416,7 @@ public async Task Vector132WireConformanceToRequestAnthropicSystemSeparate() } [Fact] - public async Task Vector133WireConformanceToRequestAnthropicToolWire() + public async Task Vector134WireConformanceToRequestAnthropicToolWire() { string vectorJson = """ { @@ -8469,7 +8503,7 @@ public async Task Vector133WireConformanceToRequestAnthropicToolWire() } [Fact] - public async Task Vector134WireConformanceToRequestChatAudioMp3() + public async Task Vector135WireConformanceToRequestChatAudioMp3() { string vectorJson = """ { @@ -8529,7 +8563,7 @@ public async Task Vector134WireConformanceToRequestChatAudioMp3() } [Fact] - public async Task Vector135WireConformanceToRequestChatAudioPart() + public async Task Vector136WireConformanceToRequestChatAudioPart() { string vectorJson = """ { @@ -8589,7 +8623,7 @@ public async Task Vector135WireConformanceToRequestChatAudioPart() } [Fact] - public async Task Vector136WireConformanceToRequestChatImageBase64() + public async Task Vector137WireConformanceToRequestChatImageBase64() { string vectorJson = """ { @@ -8655,7 +8689,7 @@ public async Task Vector136WireConformanceToRequestChatImageBase64() } [Fact] - public async Task Vector137WireConformanceToRequestChatImagePart() + public async Task Vector138WireConformanceToRequestChatImagePart() { string vectorJson = """ { @@ -8721,7 +8755,7 @@ public async Task Vector137WireConformanceToRequestChatImagePart() } [Fact] - public async Task Vector138WireConformanceToRequestChatMultipartContent() + public async Task Vector139WireConformanceToRequestChatMultipartContent() { string vectorJson = """ { @@ -8787,7 +8821,7 @@ public async Task Vector138WireConformanceToRequestChatMultipartContent() } [Fact] - public async Task Vector139WireConformanceToRequestChatSimple() + public async Task Vector140WireConformanceToRequestChatSimple() { string vectorJson = """ { @@ -8851,7 +8885,7 @@ public async Task Vector139WireConformanceToRequestChatSimple() } [Fact] - public async Task Vector140WireConformanceToRequestChatSingleTextOptimized() + public async Task Vector141WireConformanceToRequestChatSingleTextOptimized() { string vectorJson = """ { @@ -8902,7 +8936,7 @@ public async Task Vector140WireConformanceToRequestChatSingleTextOptimized() } [Fact] - public async Task Vector141WireConformanceToRequestChatWithOptions() + public async Task Vector142WireConformanceToRequestChatWithOptions() { string vectorJson = """ { @@ -8972,7 +9006,7 @@ public async Task Vector141WireConformanceToRequestChatWithOptions() } [Fact] - public async Task Vector142WireConformanceToRequestEmbeddingWire() + public async Task Vector143WireConformanceToRequestEmbeddingWire() { string vectorJson = """ { @@ -9018,7 +9052,7 @@ public async Task Vector142WireConformanceToRequestEmbeddingWire() } [Fact] - public async Task Vector143WireConformanceToRequestImageWire() + public async Task Vector144WireConformanceToRequestImageWire() { string vectorJson = """ { @@ -9064,7 +9098,7 @@ public async Task Vector143WireConformanceToRequestImageWire() } [Fact] - public async Task Vector144WireConformanceToRequestKindToJsonTypeMapping() + public async Task Vector145WireConformanceToRequestKindToJsonTypeMapping() { string vectorJson = """ { @@ -9185,7 +9219,7 @@ public async Task Vector144WireConformanceToRequestKindToJsonTypeMapping() } [Fact] - public async Task Vector145WireConformanceToRequestOptionsAdditionalProperties() + public async Task Vector146WireConformanceToRequestOptionsAdditionalProperties() { string vectorJson = """ { @@ -9241,7 +9275,7 @@ public async Task Vector145WireConformanceToRequestOptionsAdditionalProperties() } [Fact] - public async Task Vector146WireConformanceToRequestOptionsMaxCompletionTokens() + public async Task Vector147WireConformanceToRequestOptionsMaxCompletionTokens() { string vectorJson = """ { @@ -9295,7 +9329,7 @@ public async Task Vector146WireConformanceToRequestOptionsMaxCompletionTokens() } [Fact] - public async Task Vector147WireConformanceToRequestOptionsStopSequences() + public async Task Vector148WireConformanceToRequestOptionsStopSequences() { string vectorJson = """ { @@ -9355,7 +9389,7 @@ public async Task Vector147WireConformanceToRequestOptionsStopSequences() } [Fact] - public async Task Vector148WireConformanceToRequestResponsesSimple() + public async Task Vector149WireConformanceToRequestResponsesSimple() { string vectorJson = """ { @@ -9416,7 +9450,7 @@ public async Task Vector148WireConformanceToRequestResponsesSimple() } [Fact] - public async Task Vector149WireConformanceToRequestResponsesStructuredOutput() + public async Task Vector150WireConformanceToRequestResponsesStructuredOutput() { string vectorJson = """ { @@ -9505,7 +9539,7 @@ public async Task Vector149WireConformanceToRequestResponsesStructuredOutput() } [Fact] - public async Task Vector150WireConformanceToRequestResponsesWithTools() + public async Task Vector151WireConformanceToRequestResponsesWithTools() { string vectorJson = """ { @@ -9587,7 +9621,7 @@ public async Task Vector150WireConformanceToRequestResponsesWithTools() } [Fact] - public async Task Vector151WireConformanceToRequestStructuredOutput() + public async Task Vector152WireConformanceToRequestStructuredOutput() { string vectorJson = """ { @@ -9672,7 +9706,7 @@ public async Task Vector151WireConformanceToRequestStructuredOutput() } [Fact] - public async Task Vector152WireConformanceToRequestStructuredOutputNestedOptional() + public async Task Vector153WireConformanceToRequestStructuredOutputNestedOptional() { string vectorJson = """ { @@ -9776,7 +9810,7 @@ public async Task Vector152WireConformanceToRequestStructuredOutputNestedOptiona } [Fact] - public async Task Vector153WireConformanceToRequestToolsBindingsStripped() + public async Task Vector154WireConformanceToRequestToolsBindingsStripped() { string vectorJson = """ { @@ -9870,7 +9904,7 @@ public async Task Vector153WireConformanceToRequestToolsBindingsStripped() } [Fact] - public async Task Vector154WireConformanceToRequestToolsFunctionWire() + public async Task Vector155WireConformanceToRequestToolsFunctionWire() { string vectorJson = """ { @@ -9954,7 +9988,7 @@ public async Task Vector154WireConformanceToRequestToolsFunctionWire() } [Fact] - public async Task Vector155WireConformanceToRequestToolsNullWhenEmpty() + public async Task Vector156WireConformanceToRequestToolsNullWhenEmpty() { string vectorJson = """ { @@ -10005,7 +10039,7 @@ public async Task Vector155WireConformanceToRequestToolsNullWhenEmpty() } [Fact] - public async Task Vector156WireConformanceToRequestToolsStrictMode() + public async Task Vector157WireConformanceToRequestToolsStrictMode() { string vectorJson = """ { diff --git a/runtime/csharp/Prompty.Core/EndpointTrust.cs b/runtime/csharp/Prompty.Core/EndpointTrust.cs index 59bd2517a..e3a07f29e 100644 --- a/runtime/csharp/Prompty.Core/EndpointTrust.cs +++ b/runtime/csharp/Prompty.Core/EndpointTrust.cs @@ -119,7 +119,8 @@ private static bool FileSuppliedCredential(IDictionary connection) var key = Get(connection, "kind") switch { "key" => "apiKey", - "oauth" => "clientSecret", + + _ => null, }; if (key is null) diff --git a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs index 7bb2958dd..b24b0fd4b 100644 --- a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs +++ b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs @@ -583,6 +583,20 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea } if (caught is null) throw new SeamConformanceException("untrusted_endpoint_no_key_rejected: expected an error"); } + { + // vector: untrusted_endpoint_oauth_literal_secret_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_oauth_literal_secret_rejected: expected an error"); + } { // vector: untrusted_endpoint_percent_encoded_host_rejected var source = JsonSerializer.Deserialize("null")!; diff --git a/runtime/go/prompty/model/endpoint_trust.go b/runtime/go/prompty/model/endpoint_trust.go index 08e138da7..5460d52c7 100644 --- a/runtime/go/prompty/model/endpoint_trust.go +++ b/runtime/go/prompty/model/endpoint_trust.go @@ -33,7 +33,7 @@ var BuiltinTrustedEndpoints = []string{ var defaultPorts = map[string]int{"http": 80, "https": 443} // credentialKeys maps a connection kind to the credential field it consumes. -var credentialKeys = map[string]string{"key": "apiKey", "oauth": "clientSecret"} +var credentialKeys = map[string]string{"key": "apiKey"} var destinationKeys = []string{"endpoint", "tokenUrl"} diff --git a/runtime/go/prompty/model/vector_conformance_test.go b/runtime/go/prompty/model/vector_conformance_test.go index e6c3f22f4..7aa64abb6 100644 --- a/runtime/go/prompty/model/vector_conformance_test.go +++ b/runtime/go/prompty/model/vector_conformance_test.go @@ -2313,7 +2313,40 @@ func TestVector65LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected(t *testing.T) { +func TestVector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_oauth_literal_secret_rejected", + "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-oauth-literal", + "model": { + "id": "gpt-4", + "provider": "anthropic", + "connection": { + "kind": "oauth", + "endpoint": "https://attacker.example.com", + "clientId": "decoy", + "clientSecret": "decoy", + "tokenUrl": "https://login.microsoftonline.com/t/oauth2/v2.0/token" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_percent_encoded_host_rejected", "description": "Hosts are compared without percent-decoding, so an encoded host never matches", @@ -2342,7 +2375,7 @@ func TestVector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected( vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected(t *testing.T) { +func TestVector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_port_mismatch_rejected", "description": "A non-default port is a different origin", @@ -2371,7 +2404,7 @@ func TestVector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected(t *tes vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected(t *testing.T) { +func TestVector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_query_before_at_rejected", "description": "A '?' ends the authority, so the host is evil.example.com", @@ -2400,7 +2433,7 @@ func TestVector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected(t *te vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected(t *testing.T) { +func TestVector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_scheme_mismatch_rejected", "description": "Built-in origins are https; http on the same host is a different origin", @@ -2429,7 +2462,7 @@ func TestVector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected(t *t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected(t *testing.T) { +func TestVector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_suffix_lookalike_rejected", "description": "A trusted host as a label prefix of another domain is not trusted", @@ -2458,7 +2491,7 @@ func TestVector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected(t * vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *testing.T) { +func TestVector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_suffix_spoof_rejected", "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", @@ -2490,7 +2523,7 @@ func TestVector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *test vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected(t *testing.T) { +func TestVector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_trailing_dot_rejected", "description": "Hosts are compared exactly; a trailing-dot host does not match", @@ -2519,7 +2552,7 @@ func TestVector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected(t *test vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected(t *testing.T) { +func TestVector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_unicode_homoglyph_rejected", "description": "A non-ASCII lookalike host does not match", @@ -2548,7 +2581,7 @@ func TestVector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected(t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected(t *testing.T) { +func TestVector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_userinfo_lookalike_rejected", "description": "A trusted host in userinfo is not the destination host", @@ -2577,7 +2610,7 @@ func TestVector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected(t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *testing.T) { +func TestVector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_wildcard_apex_rejected", "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", @@ -2606,7 +2639,7 @@ func TestVector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *tes vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected(t *testing.T) { +func TestVector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_wildcard_boundary_rejected", "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", @@ -2635,7 +2668,7 @@ func TestVector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected(t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector77MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { +func TestVector78MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorJSON := `{ "name": "clear_by_category_and_all", "description": "Clear removes only the requested category, or all memories when no category is supplied.", @@ -2688,7 +2721,7 @@ func TestVector77MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector78MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { +func TestVector79MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { vectorJSON := `{ "name": "format_core_and_recall_results", "description": "Formatting injects only core memories into system prompts and formats recall results with category and tags.", @@ -2729,7 +2762,7 @@ func TestVector78MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector79MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { +func TestVector80MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { vectorJSON := `{ "name": "recall_empty_query_and_limit", "description": "Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.", @@ -2783,7 +2816,7 @@ func TestVector79MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector80MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { +func TestVector81MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { vectorJSON := `{ "name": "recall_ranks_by_weighted_score", "description": "Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.", @@ -2841,7 +2874,7 @@ func TestVector80MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { +func TestVector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { vectorJSON := `{ "name": "recall_score_ties_preserve_insertion_order", "description": "Recall preserves insertion order for memories with identical scores.", @@ -2898,7 +2931,7 @@ func TestVector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector82MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { +func TestVector83MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorJSON := `{ "name": "recall_tags_and_core_boost", "description": "Recall gives tag matches higher weight than content matches and boosts matching core memories.", @@ -2958,7 +2991,7 @@ func TestVector82MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { +func TestVector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { vectorJSON := `{ "name": "recall_tokenization_punctuation_unicode_and_dedup", "description": "Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.", @@ -3018,7 +3051,7 @@ func TestVector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAnd vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector84MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { +func TestVector85MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_by_tags", "description": "Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.", @@ -3081,7 +3114,7 @@ func TestVector84MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { +func TestVector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_missing_and_empty_tags", "description": "Remember treats missing and empty tag lists as equivalent for core-memory deduplication.", @@ -3136,7 +3169,7 @@ func TestVector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector86MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { +func TestVector87MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { vectorJSON := `{ "name": "remember_evicts_archival_first", "description": "Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.", @@ -3196,7 +3229,7 @@ func TestVector86MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing. vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector87MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { +func TestVector88MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorJSON := `{ "name": "snapshot_port_round_trip", "description": "A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.", @@ -3235,7 +3268,7 @@ func TestVector87MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector88MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { +func TestVector89MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { vectorJSON := `{ "name": "update_index_out_of_range_error", "description": "Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.", @@ -3268,7 +3301,7 @@ func TestVector88MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector89TurnConformanceReplayMaxIterations(t *testing.T) { +func TestVector90TurnConformanceReplayMaxIterations(t *testing.T) { vectorJSON := `{ "name": "max_iterations", "stage": "replay", @@ -3304,7 +3337,7 @@ func TestVector89TurnConformanceReplayMaxIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector90TurnConformanceReplayNoTool(t *testing.T) { +func TestVector91TurnConformanceReplayNoTool(t *testing.T) { vectorJSON := `{ "name": "no_tool", "stage": "replay", @@ -3336,7 +3369,7 @@ func TestVector90TurnConformanceReplayNoTool(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector91TurnConformanceReplayPermissionDenied(t *testing.T) { +func TestVector92TurnConformanceReplayPermissionDenied(t *testing.T) { vectorJSON := `{ "name": "permission_denied", "stage": "replay", @@ -3371,7 +3404,7 @@ func TestVector91TurnConformanceReplayPermissionDenied(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector92TurnConformanceReplayToolFailure(t *testing.T) { +func TestVector93TurnConformanceReplayToolFailure(t *testing.T) { vectorJSON := `{ "name": "tool_failure", "stage": "replay", @@ -3408,7 +3441,7 @@ func TestVector92TurnConformanceReplayToolFailure(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector93TurnConformanceReplayToolSuccess(t *testing.T) { +func TestVector94TurnConformanceReplayToolSuccess(t *testing.T) { vectorJSON := `{ "name": "tool_success", "stage": "replay", @@ -3445,7 +3478,7 @@ func TestVector93TurnConformanceReplayToolSuccess(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector94TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { +func TestVector95TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorJSON := `{ "name": "assistant_tool_calls_metadata", "description": "When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.", @@ -3598,7 +3631,7 @@ func TestVector94TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector95TurnConformanceRunAsyncToolFunction(t *testing.T) { +func TestVector96TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorJSON := `{ "name": "async_tool_function", "description": "Agent loop correctly handles async tool functions", @@ -3709,7 +3742,7 @@ func TestVector95TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector96TurnConformanceRunBindingsInjected(t *testing.T) { +func TestVector97TurnConformanceRunBindingsInjected(t *testing.T) { vectorJSON := `{ "name": "bindings_injected", "description": "Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.", @@ -3840,7 +3873,7 @@ func TestVector96TurnConformanceRunBindingsInjected(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector97TurnConformanceRunCancellationBeforeLlm(t *testing.T) { +func TestVector98TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorJSON := `{ "name": "cancellation_before_llm", "description": "§13.2 Cancellation — Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.", @@ -3902,7 +3935,7 @@ func TestVector97TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector98TurnConformanceRunCancellationBetweenIterations(t *testing.T) { +func TestVector99TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_iterations", "description": "§13.2 Cancellation — Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.", @@ -4033,7 +4066,7 @@ func TestVector98TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector99TurnConformanceRunCancellationBetweenTools(t *testing.T) { +func TestVector100TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_tools", "description": "§13.2 Cancellation — LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.", @@ -4173,7 +4206,7 @@ func TestVector99TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector100TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { +func TestVector101TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorJSON := `{ "name": "context_no_trim_when_fits", "description": "§13.3 Context Window — Messages fit within the context budget. No trimming occurs and all messages are preserved.", @@ -4245,7 +4278,7 @@ func TestVector100TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector101TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { +func TestVector102TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { vectorJSON := `{ "name": "context_preserves_system_messages", "description": "§13.3 Context Window — Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.", @@ -4353,7 +4386,7 @@ func TestVector101TurnConformanceRunContextPreservesSystemMessages(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector102TurnConformanceRunContextTrimBasic(t *testing.T) { +func TestVector103TurnConformanceRunContextTrimBasic(t *testing.T) { vectorJSON := `{ "name": "context_trim_basic", "description": "§13.3 Context Window — Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.", @@ -4512,7 +4545,7 @@ func TestVector102TurnConformanceRunContextTrimBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector103TurnConformanceRunEmptyToolResult(t *testing.T) { +func TestVector104TurnConformanceRunEmptyToolResult(t *testing.T) { vectorJSON := `{ "name": "empty_tool_result", "description": "Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content — it must not be skipped or filtered out.", @@ -4628,7 +4661,7 @@ func TestVector103TurnConformanceRunEmptyToolResult(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector104TurnConformanceRunEventsBasicToolLoop(t *testing.T) { +func TestVector105TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorJSON := `{ "name": "events_basic_tool_loop", "description": "§13.1 Events — A 2-turn tool-call loop emits the full event lifecycle: status → tool_call_start → tool_result → messages_updated → done.", @@ -4775,7 +4808,7 @@ func TestVector104TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector105TurnConformanceRunEventsErrorLogged(t *testing.T) { +func TestVector106TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorJSON := `{ "name": "events_error_logged", "description": "§13.1 Events — A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.", @@ -4902,7 +4935,7 @@ func TestVector105TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector106TurnConformanceRunEventsNoTools(t *testing.T) { +func TestVector107TurnConformanceRunEventsNoTools(t *testing.T) { vectorJSON := `{ "name": "events_no_tools", "description": "§13.1 Events — Single-turn completion with no tool calls. Only the done event is emitted.", @@ -4981,7 +5014,7 @@ func TestVector106TurnConformanceRunEventsNoTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector107TurnConformanceRunGuardrailAllPass(t *testing.T) { +func TestVector108TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorJSON := `{ "name": "guardrail_all_pass", "description": "§13.4 Guardrails — All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.", @@ -5105,7 +5138,7 @@ func TestVector107TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector108TurnConformanceRunGuardrailInputDeny(t *testing.T) { +func TestVector109TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_input_deny", "description": "§13.4 Guardrails — Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.", @@ -5162,7 +5195,7 @@ func TestVector108TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector109TurnConformanceRunGuardrailOutputDeny(t *testing.T) { +func TestVector110TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_output_deny", "description": "§13.4 Guardrails — Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.", @@ -5241,7 +5274,7 @@ func TestVector109TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector110TurnConformanceRunGuardrailToolDeny(t *testing.T) { +func TestVector111TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_tool_deny", "description": "§13.4 Guardrails — LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.", @@ -5396,7 +5429,7 @@ func TestVector110TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector111TurnConformanceRunMaxIterationsExceeded(t *testing.T) { +func TestVector112TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorJSON := `{ "name": "max_iterations_exceeded", "description": "LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.", @@ -5907,7 +5940,7 @@ func TestVector111TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector112TurnConformanceRunMultiTurnToolCalls(t *testing.T) { +func TestVector113TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorJSON := `{ "name": "multi_turn_tool_calls", "description": "Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.", @@ -6085,7 +6118,7 @@ func TestVector112TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector113TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { +func TestVector114TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorJSON := `{ "name": "multiple_tool_calls_single_turn", "description": "Turn 1: LLM returns two parallel tool calls — get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.", @@ -6267,7 +6300,7 @@ func TestVector113TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector114TurnConformanceRunNoToolCalls(t *testing.T) { +func TestVector115TurnConformanceRunNoToolCalls(t *testing.T) { vectorJSON := `{ "name": "no_tool_calls", "description": "LLM returns content immediately with no tool calls — the agent loop completes in a single iteration.", @@ -6336,7 +6369,7 @@ func TestVector114TurnConformanceRunNoToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector115TurnConformanceRunParallelToolsBasic(t *testing.T) { +func TestVector116TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_basic", "description": "§13.6 Parallel Tools — LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.", @@ -6510,7 +6543,7 @@ func TestVector115TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector116TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { +func TestVector117TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_guardrail_deny", "description": "§13.6 Parallel Tools + §13.4 Guardrails — 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.", @@ -6702,7 +6735,7 @@ func TestVector116TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector117TurnConformanceRunParallelToolsWithToolError(t *testing.T) { +func TestVector118TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_tool_error", "description": "§13.6 Parallel Tools + tool error — 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.", @@ -6876,7 +6909,7 @@ func TestVector117TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector118TurnConformanceRunSingleToolCall(t *testing.T) { +func TestVector119TurnConformanceRunSingleToolCall(t *testing.T) { vectorJSON := `{ "name": "single_tool_call", "description": "Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.", @@ -7024,7 +7057,7 @@ func TestVector118TurnConformanceRunSingleToolCall(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector119TurnConformanceRunSteeringInjectMessage(t *testing.T) { +func TestVector120TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorJSON := `{ "name": "steering_inject_message", "description": "§13.5 Steering — After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.", @@ -7233,7 +7266,7 @@ func TestVector119TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector120TurnConformanceRunSteeringMultipleMessages(t *testing.T) { +func TestVector121TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorJSON := `{ "name": "steering_multiple_messages", "description": "§13.5 Steering — Two steering messages are queued and both injected before iteration 2. Tests atomic drain — both appear in order.", @@ -7404,7 +7437,7 @@ func TestVector120TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector121TurnConformanceRunToolNotRegisteredError(t *testing.T) { +func TestVector122TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorJSON := `{ "name": "tool_not_registered_error", "description": "LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.", @@ -7489,7 +7522,7 @@ func TestVector121TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector122TurnConformanceRunToolResultMessageFormat(t *testing.T) { +func TestVector123TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorJSON := `{ "name": "tool_result_message_format", "description": "After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.", @@ -7611,7 +7644,7 @@ func TestVector122TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector123TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { +func TestVector124TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorJSON := `{ "name": "cancel_before_context", "stage": "turn", @@ -7644,7 +7677,7 @@ func TestVector123TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector124TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { +func TestVector125TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorJSON := `{ "name": "delegated_provider_state", "stage": "turn", @@ -7712,7 +7745,7 @@ func TestVector124TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector125TurnConformanceRunTurnFinalOutput(t *testing.T) { +func TestVector126TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorJSON := `{ "name": "final_output", "stage": "turn", @@ -7758,7 +7791,7 @@ func TestVector125TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector126TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { +func TestVector127TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "memory_context_is_model_visible", "stage": "turn", @@ -7821,7 +7854,7 @@ func TestVector126TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector127TurnConformanceRunTurnOrderedToolRound(t *testing.T) { +func TestVector128TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorJSON := `{ "name": "ordered_tool_round", "stage": "turn", @@ -7913,7 +7946,7 @@ func TestVector127TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector128TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { +func TestVector129TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "permission_denial_is_model_visible", "stage": "turn", @@ -7965,7 +7998,7 @@ func TestVector128TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testin vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector129WireConformanceToRequestAnthropicImageFormat(t *testing.T) { +func TestVector130WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorJSON := `{ "name": "anthropic_image_format", "description": "§7.5 — Anthropic: ImagePart uses base64 source block with media_type, not image_url format.", @@ -8036,7 +8069,7 @@ func TestVector129WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector130WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { +func TestVector131WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { vectorJSON := `{ "name": "anthropic_max_tokens_required", "description": "§7.5 — Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).", @@ -8092,7 +8125,7 @@ func TestVector130WireConformanceToRequestAnthropicMaxTokensRequired(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector131WireConformanceToRequestAnthropicOptions(t *testing.T) { +func TestVector132WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorJSON := `{ "name": "anthropic_options", "description": "Anthropic: option names map correctly (topK, stopSequences)", @@ -8162,7 +8195,7 @@ func TestVector131WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector132WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { +func TestVector133WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { vectorJSON := `{ "name": "anthropic_system_separate", "description": "§7.5 — Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.", @@ -8230,7 +8263,7 @@ func TestVector132WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector133WireConformanceToRequestAnthropicToolWire(t *testing.T) { +func TestVector134WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorJSON := `{ "name": "anthropic_tool_wire", "description": "Anthropic: tools use input_schema (not parameters), name at top level", @@ -8316,7 +8349,7 @@ func TestVector133WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector134WireConformanceToRequestChatAudioMp3(t *testing.T) { +func TestVector135WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorJSON := `{ "name": "chat_audio_mp3", "description": "§7.1.2 — AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.", @@ -8375,7 +8408,7 @@ func TestVector134WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector135WireConformanceToRequestChatAudioPart(t *testing.T) { +func TestVector136WireConformanceToRequestChatAudioPart(t *testing.T) { vectorJSON := `{ "name": "chat_audio_part", "description": "§7.1.2 — AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.", @@ -8434,7 +8467,7 @@ func TestVector135WireConformanceToRequestChatAudioPart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector136WireConformanceToRequestChatImageBase64(t *testing.T) { +func TestVector137WireConformanceToRequestChatImageBase64(t *testing.T) { vectorJSON := `{ "name": "chat_image_base64", "description": "Base64 data URI image part preserved as-is in image_url", @@ -8499,7 +8532,7 @@ func TestVector136WireConformanceToRequestChatImageBase64(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector137WireConformanceToRequestChatImagePart(t *testing.T) { +func TestVector138WireConformanceToRequestChatImagePart(t *testing.T) { vectorJSON := `{ "name": "chat_image_part", "description": "Image content part converted to OpenAI image_url format", @@ -8564,7 +8597,7 @@ func TestVector137WireConformanceToRequestChatImagePart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector138WireConformanceToRequestChatMultipartContent(t *testing.T) { +func TestVector139WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorJSON := `{ "name": "chat_multipart_content", "description": "§7.1.1/§7.1.2 — Message with TextPart + ImagePart MUST produce an array of typed content blocks.", @@ -8629,7 +8662,7 @@ func TestVector138WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector139WireConformanceToRequestChatSimple(t *testing.T) { +func TestVector140WireConformanceToRequestChatSimple(t *testing.T) { vectorJSON := `{ "name": "chat_simple", "description": "§7.1.7 — Two messages (system + user), no tools, no options. Minimal chat request.", @@ -8692,7 +8725,7 @@ func TestVector139WireConformanceToRequestChatSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector140WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { +func TestVector141WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { vectorJSON := `{ "name": "chat_single_text_optimized", "description": "§7.1.1 — Message with exactly 1 TextPart MUST produce a plain string content, not an array.", @@ -8742,7 +8775,7 @@ func TestVector140WireConformanceToRequestChatSingleTextOptimized(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector141WireConformanceToRequestChatWithOptions(t *testing.T) { +func TestVector142WireConformanceToRequestChatWithOptions(t *testing.T) { vectorJSON := `{ "name": "chat_with_options", "description": "§7.1.5 — All standard ModelOptions mapped to OpenAI parameter names.", @@ -8811,7 +8844,7 @@ func TestVector141WireConformanceToRequestChatWithOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector142WireConformanceToRequestEmbeddingWire(t *testing.T) { +func TestVector143WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorJSON := `{ "name": "embedding_wire", "description": "§7.2 — Embedding request: text extracted from messages, single string input (not array).", @@ -8856,7 +8889,7 @@ func TestVector142WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector143WireConformanceToRequestImageWire(t *testing.T) { +func TestVector144WireConformanceToRequestImageWire(t *testing.T) { vectorJSON := `{ "name": "image_wire", "description": "§7.3 — Image generation request: prompt extracted from last user message.", @@ -8901,7 +8934,7 @@ func TestVector143WireConformanceToRequestImageWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector144WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { +func TestVector145WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorJSON := `{ "name": "kind_to_json_type_mapping", "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.", @@ -9021,7 +9054,7 @@ func TestVector144WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector145WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { +func TestVector146WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { vectorJSON := `{ "name": "options_additional_properties", "description": "§7.1.5 — additionalProperties from ModelOptions MUST be merged into the request as top-level keys.", @@ -9076,7 +9109,7 @@ func TestVector145WireConformanceToRequestOptionsAdditionalProperties(t *testing vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector146WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { +func TestVector147WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { vectorJSON := `{ "name": "options_max_completion_tokens", "description": "§7.1.5 — maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.", @@ -9129,7 +9162,7 @@ func TestVector146WireConformanceToRequestOptionsMaxCompletionTokens(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector147WireConformanceToRequestOptionsStopSequences(t *testing.T) { +func TestVector148WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorJSON := `{ "name": "options_stop_sequences", "description": "§7.1.5 — stopSequences MUST map to 'stop' in the OpenAI wire format.", @@ -9188,7 +9221,7 @@ func TestVector147WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector148WireConformanceToRequestResponsesSimple(t *testing.T) { +func TestVector149WireConformanceToRequestResponsesSimple(t *testing.T) { vectorJSON := `{ "name": "responses_simple", "description": "Responses API: system becomes instructions, messages become input items", @@ -9248,7 +9281,7 @@ func TestVector148WireConformanceToRequestResponsesSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector149WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { +func TestVector150WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "responses_structured_output", "description": "Responses API: structured output uses text.format.json_schema", @@ -9336,7 +9369,7 @@ func TestVector149WireConformanceToRequestResponsesStructuredOutput(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector150WireConformanceToRequestResponsesWithTools(t *testing.T) { +func TestVector151WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorJSON := `{ "name": "responses_with_tools", "description": "Responses API: tools use flat format (no nested function key)", @@ -9417,7 +9450,7 @@ func TestVector150WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector151WireConformanceToRequestStructuredOutput(t *testing.T) { +func TestVector152WireConformanceToRequestStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "structured_output", "description": "§7.1.6 — outputs converted to response_format with json_schema, strict=true, additionalProperties=false.", @@ -9501,7 +9534,7 @@ func TestVector151WireConformanceToRequestStructuredOutput(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector152WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { +func TestVector153WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { vectorJSON := `{ "name": "structured_output_nested_optional", "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", @@ -9604,7 +9637,7 @@ func TestVector152WireConformanceToRequestStructuredOutputNestedOptional(t *test vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector153WireConformanceToRequestToolsBindingsStripped(t *testing.T) { +func TestVector154WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorJSON := `{ "name": "tools_bindings_stripped", "description": "§7.1.3 — Parameters listed in bindings MUST be stripped from wire tools (properties AND required).", @@ -9697,7 +9730,7 @@ func TestVector153WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector154WireConformanceToRequestToolsFunctionWire(t *testing.T) { +func TestVector155WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorJSON := `{ "name": "tools_function_wire", "description": "§7.1.3 — FunctionTool projected as OpenAI function definition with JSON Schema parameters.", @@ -9780,7 +9813,7 @@ func TestVector154WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector155WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { +func TestVector156WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorJSON := `{ "name": "tools_null_when_empty", "description": "§7.1.3 — When no tools are defined, the tools key MUST be absent from the request entirely.", @@ -9830,7 +9863,7 @@ func TestVector155WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector156WireConformanceToRequestToolsStrictMode(t *testing.T) { +func TestVector157WireConformanceToRequestToolsStrictMode(t *testing.T) { vectorJSON := `{ "name": "tools_strict_mode", "description": "§7.1.3 — FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.", diff --git a/runtime/go/prompty/model/vectorconformance/vector_conformance.go b/runtime/go/prompty/model/vectorconformance/vector_conformance.go index e50d1b317..db81ac528 100644 --- a/runtime/go/prompty/model/vectorconformance/vector_conformance.go +++ b/runtime/go/prompty/model/vectorconformance/vector_conformance.go @@ -1405,6 +1405,17 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("untrusted_endpoint_no_key_rejected: expected an error, got nil") } } + // vector: untrusted_endpoint_oauth_literal_secret_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_oauth_literal_secret_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_oauth_literal_secret_rejected: expected an error, got nil") + } + } // vector: untrusted_endpoint_percent_encoded_host_rejected { var source interface{} diff --git a/runtime/java/README.md b/runtime/java/README.md index 91f3c49b8..3ef0b6966 100644 --- a/runtime/java/README.md +++ b/runtime/java/README.md @@ -66,8 +66,7 @@ cd runtime/java ## Endpoint trust A `.prompty` file is untrusted input. When its `model.connection` carries no -file-supplied credential (a literal `apiKey` for `kind: key`, a literal -`clientSecret` for `kind: oauth`), executors fall back to ambient credentials, +file-supplied credential (a literal `apiKey` on `kind: key`), executors fall back to ambient credentials, so the loader refuses — with `LoadException.Kind.UNTRUSTED_ENDPOINT` — any file-chosen `endpoint` / `tokenUrl` whose origin is not trusted (spec §2.5.1). Built-in provider origins are always trusted; hosts add more through diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java index 99580a6f3..0116b710c 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java @@ -34,7 +34,7 @@ public final class EndpointTrust { "https://login.microsoftonline.com"); private static final Map CREDENTIAL_KEYS = - Map.of("key", "apiKey", "oauth", "clientSecret"); + Map.of("key", "apiKey"); private static final List DESTINATION_KEYS = List.of("endpoint", "tokenUrl"); private EndpointTrust() {} diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java index 66c82ffec..513eae313 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java @@ -844,6 +844,20 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex } if (caught == null) throw new AssertionError("untrusted_endpoint_no_key_rejected: expected an error"); } + { + // vector: untrusted_endpoint_oauth_literal_secret_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_oauth_literal_secret_rejected: expected an error"); + } { // vector: untrusted_endpoint_percent_encoded_host_rejected StringBuilder inputJson = new StringBuilder(); diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java index a1c756148..8591718e1 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java @@ -519,637 +519,644 @@ private static void vector65LoadConformanceLoadUntrustedEndpointNoKeyRejected() VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() throws Exception { + private static void vector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_oauth_literal_secret_rejected\",\"description\":\"kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_percent_encoded_host_rejected\",\"description\":\"Hosts are compared without percent-decoding, so an encoded host never matches\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected() throws Exception { + private static void vector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_port_mismatch_rejected\",\"description\":\"A non-default port is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() throws Exception { + private static void vector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_query_before_at_rejected\",\"description\":\"A '?' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() throws Exception { + private static void vector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_scheme_mismatch_rejected\",\"description\":\"Built-in origins are https; http on the same host is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() throws Exception { + private static void vector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_suffix_lookalike_rejected\",\"description\":\"A trusted host as a label prefix of another domain is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() throws Exception { + private static void vector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected() throws Exception { + private static void vector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_trailing_dot_rejected\",\"description\":\"Hosts are compared exactly; a trailing-dot host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() throws Exception { + private static void vector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_unicode_homoglyph_rejected\",\"description\":\"A non-ASCII lookalike host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\\u0430pi.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() throws Exception { + private static void vector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_userinfo_lookalike_rejected\",\"description\":\"A trusted host in userinfo is not the destination host\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected() throws Exception { + private static void vector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() throws Exception { + private static void vector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_wildcard_boundary_rejected\",\"description\":\"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector77MemoryConformanceOperateClearByCategoryAndAll() throws Exception { + private static void vector78MemoryConformanceOperateClearByCategoryAndAll() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector78MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { + private static void vector79MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector79MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { + private static void vector80MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector80MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { + private static void vector81MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { + private static void vector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector82MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { + private static void vector83MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { + private static void vector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\\u00e9\"]},{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\\u00e9\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector84MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { + private static void vector85MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { + private static void vector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector86MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { + private static void vector87MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector87MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { + private static void vector88MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector88MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { + private static void vector89MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector89TurnConformanceReplayMaxIterations() throws Exception { + private static void vector90TurnConformanceReplayMaxIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector90TurnConformanceReplayNoTool() throws Exception { + private static void vector91TurnConformanceReplayNoTool() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector91TurnConformanceReplayPermissionDenied() throws Exception { + private static void vector92TurnConformanceReplayPermissionDenied() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector92TurnConformanceReplayToolFailure() throws Exception { + private static void vector93TurnConformanceReplayToolFailure() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector93TurnConformanceReplayToolSuccess() throws Exception { + private static void vector94TurnConformanceReplayToolSuccess() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector94TurnConformanceRunAssistantToolCallsMetadata() throws Exception { + private static void vector95TurnConformanceRunAssistantToolCallsMetadata() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector95TurnConformanceRunAsyncToolFunction() throws Exception { + private static void vector96TurnConformanceRunAsyncToolFunction() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector96TurnConformanceRunBindingsInjected() throws Exception { + private static void vector97TurnConformanceRunBindingsInjected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\\u00b0C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector97TurnConformanceRunCancellationBeforeLlm() throws Exception { + private static void vector98TurnConformanceRunCancellationBeforeLlm() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_before_llm\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector98TurnConformanceRunCancellationBetweenIterations() throws Exception { + private static void vector99TurnConformanceRunCancellationBetweenIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_iterations\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector99TurnConformanceRunCancellationBetweenTools() throws Exception { + private static void vector100TurnConformanceRunCancellationBetweenTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_tools\",\"description\":\"\\u00a713.2 Cancellation \\u2014 LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector100TurnConformanceRunContextNoTrimWhenFits() throws Exception { + private static void vector101TurnConformanceRunContextNoTrimWhenFits() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector101TurnConformanceRunContextPreservesSystemMessages() throws Exception { + private static void vector102TurnConformanceRunContextPreservesSystemMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_preserves_system_messages\",\"description\":\"\\u00a713.3 Context Window \\u2014 Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \\u2014 they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector102TurnConformanceRunContextTrimBasic() throws Exception { + private static void vector103TurnConformanceRunContextTrimBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_trim_basic\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector103TurnConformanceRunEmptyToolResult() throws Exception { + private static void vector104TurnConformanceRunEmptyToolResult() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \\u2014 it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector104TurnConformanceRunEventsBasicToolLoop() throws Exception { + private static void vector105TurnConformanceRunEventsBasicToolLoop() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_basic_tool_loop\",\"description\":\"\\u00a713.1 Events \\u2014 A 2-turn tool-call loop emits the full event lifecycle: status \\u2192 tool_call_start \\u2192 tool_result \\u2192 messages_updated \\u2192 done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\\u00b0F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector105TurnConformanceRunEventsErrorLogged() throws Exception { + private static void vector106TurnConformanceRunEventsErrorLogged() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_error_logged\",\"description\":\"\\u00a713.1 Events \\u0097 A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \\u0097 this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector106TurnConformanceRunEventsNoTools() throws Exception { + private static void vector107TurnConformanceRunEventsNoTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_no_tools\",\"description\":\"\\u00a713.1 Events \\u2014 Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector107TurnConformanceRunGuardrailAllPass() throws Exception { + private static void vector108TurnConformanceRunGuardrailAllPass() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_all_pass\",\"description\":\"\\u00a713.4 Guardrails \\u2014 All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector108TurnConformanceRunGuardrailInputDeny() throws Exception { + private static void vector109TurnConformanceRunGuardrailInputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_input_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector109TurnConformanceRunGuardrailOutputDeny() throws Exception { + private static void vector110TurnConformanceRunGuardrailOutputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_output_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector110TurnConformanceRunGuardrailToolDeny() throws Exception { + private static void vector111TurnConformanceRunGuardrailToolDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_tool_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \\u2014 its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector111TurnConformanceRunMaxIterationsExceeded() throws Exception { + private static void vector112TurnConformanceRunMaxIterationsExceeded() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\\u00b0F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\\u00b0F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\\u00b0F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\\u00b0F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\\u00b0F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\\u00b0F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\\u00b0F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\\u00b0F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\\u00b0F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector112TurnConformanceRunMultiTurnToolCalls() throws Exception { + private static void vector113TurnConformanceRunMultiTurnToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\\u00b0C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector113TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { + private static void vector114TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \\u2014 get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\\u00b0F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector114TurnConformanceRunNoToolCalls() throws Exception { + private static void vector115TurnConformanceRunNoToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \\u2014 the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector115TurnConformanceRunParallelToolsBasic() throws Exception { + private static void vector116TurnConformanceRunParallelToolsBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_basic\",\"description\":\"\\u00a713.6 Parallel Tools \\u2014 LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector116TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { + private static void vector117TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\\u00a713.6 Parallel Tools + \\u00a713.4 Guardrails \\u2014 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector117TurnConformanceRunParallelToolsWithToolError() throws Exception { + private static void vector118TurnConformanceRunParallelToolsWithToolError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\\u00a713.6 Parallel Tools + tool error \\u2014 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector118TurnConformanceRunSingleToolCall() throws Exception { + private static void vector119TurnConformanceRunSingleToolCall() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector119TurnConformanceRunSteeringInjectMessage() throws Exception { + private static void vector120TurnConformanceRunSteeringInjectMessage() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_inject_message\",\"description\":\"\\u00a713.5 Steering \\u2014 After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector120TurnConformanceRunSteeringMultipleMessages() throws Exception { + private static void vector121TurnConformanceRunSteeringMultipleMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_multiple_messages\",\"description\":\"\\u00a713.5 Steering \\u2014 Two steering messages are queued and both injected before iteration 2. Tests atomic drain \\u2014 both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\\u00b0F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector121TurnConformanceRunToolNotRegisteredError() throws Exception { + private static void vector122TurnConformanceRunToolNotRegisteredError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector122TurnConformanceRunToolResultMessageFormat() throws Exception { + private static void vector123TurnConformanceRunToolResultMessageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\\u00b0F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\\u00b0F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector123TurnConformanceRunTurnCancelBeforeContext() throws Exception { + private static void vector124TurnConformanceRunTurnCancelBeforeContext() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector124TurnConformanceRunTurnDelegatedProviderState() throws Exception { + private static void vector125TurnConformanceRunTurnDelegatedProviderState() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector125TurnConformanceRunTurnFinalOutput() throws Exception { + private static void vector126TurnConformanceRunTurnFinalOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector126TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { + private static void vector127TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector127TurnConformanceRunTurnOrderedToolRound() throws Exception { + private static void vector128TurnConformanceRunTurnOrderedToolRound() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector128TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { + private static void vector129TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector129WireConformanceToRequestAnthropicImageFormat() throws Exception { + private static void vector130WireConformanceToRequestAnthropicImageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_image_format\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector130WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { + private static void vector131WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector131WireConformanceToRequestAnthropicOptions() throws Exception { + private static void vector132WireConformanceToRequestAnthropicOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector132WireConformanceToRequestAnthropicSystemSeparate() throws Exception { + private static void vector133WireConformanceToRequestAnthropicSystemSeparate() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_system_separate\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector133WireConformanceToRequestAnthropicToolWire() throws Exception { + private static void vector134WireConformanceToRequestAnthropicToolWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector134WireConformanceToRequestChatAudioMp3() throws Exception { + private static void vector135WireConformanceToRequestChatAudioMp3() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_mp3\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector135WireConformanceToRequestChatAudioPart() throws Exception { + private static void vector136WireConformanceToRequestChatAudioPart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_part\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector136WireConformanceToRequestChatImageBase64() throws Exception { + private static void vector137WireConformanceToRequestChatImageBase64() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector137WireConformanceToRequestChatImagePart() throws Exception { + private static void vector138WireConformanceToRequestChatImagePart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector138WireConformanceToRequestChatMultipartContent() throws Exception { + private static void vector139WireConformanceToRequestChatMultipartContent() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_multipart_content\",\"description\":\"\\u00a77.1.1/\\u00a77.1.2 \\u2014 Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector139WireConformanceToRequestChatSimple() throws Exception { + private static void vector140WireConformanceToRequestChatSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_simple\",\"description\":\"\\u00a77.1.7 \\u2014 Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector140WireConformanceToRequestChatSingleTextOptimized() throws Exception { + private static void vector141WireConformanceToRequestChatSingleTextOptimized() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_single_text_optimized\",\"description\":\"\\u00a77.1.1 \\u2014 Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector141WireConformanceToRequestChatWithOptions() throws Exception { + private static void vector142WireConformanceToRequestChatWithOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_with_options\",\"description\":\"\\u00a77.1.5 \\u2014 All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector142WireConformanceToRequestEmbeddingWire() throws Exception { + private static void vector143WireConformanceToRequestEmbeddingWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"embedding_wire\",\"description\":\"\\u00a77.2 \\u2014 Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector143WireConformanceToRequestImageWire() throws Exception { + private static void vector144WireConformanceToRequestImageWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"image_wire\",\"description\":\"\\u00a77.3 \\u2014 Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector144WireConformanceToRequestKindToJsonTypeMapping() throws Exception { + private static void vector145WireConformanceToRequestKindToJsonTypeMapping() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\\u00a77.1.4 \\u2014 All Property kind values mapped to JSON Schema types: string\\u2192string, integer\\u2192integer, float\\u2192number, boolean\\u2192boolean, array\\u2192array, object\\u2192object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector145WireConformanceToRequestOptionsAdditionalProperties() throws Exception { + private static void vector146WireConformanceToRequestOptionsAdditionalProperties() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_additional_properties\",\"description\":\"\\u00a77.1.5 \\u2014 additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector146WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { + private static void vector147WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_max_completion_tokens\",\"description\":\"\\u00a77.1.5 \\u2014 maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector147WireConformanceToRequestOptionsStopSequences() throws Exception { + private static void vector148WireConformanceToRequestOptionsStopSequences() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_stop_sequences\",\"description\":\"\\u00a77.1.5 \\u2014 stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector148WireConformanceToRequestResponsesSimple() throws Exception { + private static void vector149WireConformanceToRequestResponsesSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector149WireConformanceToRequestResponsesStructuredOutput() throws Exception { + private static void vector150WireConformanceToRequestResponsesStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector150WireConformanceToRequestResponsesWithTools() throws Exception { + private static void vector151WireConformanceToRequestResponsesWithTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector151WireConformanceToRequestStructuredOutput() throws Exception { + private static void vector152WireConformanceToRequestStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output\",\"description\":\"\\u00a77.1.6 \\u2014 outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector152WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { + private static void vector153WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output_nested_optional\",\"description\":\"\\u00a77.1.4/\\u00a77.1.6 \\u2014 OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector153WireConformanceToRequestToolsBindingsStripped() throws Exception { + private static void vector154WireConformanceToRequestToolsBindingsStripped() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_bindings_stripped\",\"description\":\"\\u00a77.1.3 \\u2014 Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector154WireConformanceToRequestToolsFunctionWire() throws Exception { + private static void vector155WireConformanceToRequestToolsFunctionWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_function_wire\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector155WireConformanceToRequestToolsNullWhenEmpty() throws Exception { + private static void vector156WireConformanceToRequestToolsNullWhenEmpty() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_null_when_empty\",\"description\":\"\\u00a77.1.3 \\u2014 When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector156WireConformanceToRequestToolsStrictMode() throws Exception { + private static void vector157WireConformanceToRequestToolsStrictMode() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_strict_mode\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); @@ -1224,97 +1231,98 @@ public static void run() { failed += runCase("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", VectorConformanceTests::vector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected); failed += runCase("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", VectorConformanceTests::vector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected); failed += runCase("LoadConformance.load:untrusted_endpoint_no_key_rejected", VectorConformanceTests::vector65LoadConformanceLoadUntrustedEndpointNoKeyRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_percent_encoded_host_rejected", VectorConformanceTests::vector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_port_mismatch_rejected", VectorConformanceTests::vector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_query_before_at_rejected", VectorConformanceTests::vector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_scheme_mismatch_rejected", VectorConformanceTests::vector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_lookalike_rejected", VectorConformanceTests::vector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", VectorConformanceTests::vector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_trailing_dot_rejected", VectorConformanceTests::vector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_unicode_homoglyph_rejected", VectorConformanceTests::vector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_userinfo_lookalike_rejected", VectorConformanceTests::vector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", VectorConformanceTests::vector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_boundary_rejected", VectorConformanceTests::vector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected); - failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector77MemoryConformanceOperateClearByCategoryAndAll); - failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector78MemoryConformanceOperateFormatCoreAndRecallResults); - failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector79MemoryConformanceOperateRecallEmptyQueryAndLimit); - failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector80MemoryConformanceOperateRecallRanksByWeightedScore); - failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); - failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector82MemoryConformanceOperateRecallTagsAndCoreBoost); - failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); - failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector84MemoryConformanceOperateRememberDedupsCoreByTags); - failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); - failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector86MemoryConformanceOperateRememberEvictsArchivalFirst); - failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector87MemoryConformanceOperateSnapshotPortRoundTrip); - failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector88MemoryConformanceOperateUpdateIndexOutOfRangeError); - failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector89TurnConformanceReplayMaxIterations); - failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector90TurnConformanceReplayNoTool); - failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector91TurnConformanceReplayPermissionDenied); - failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector92TurnConformanceReplayToolFailure); - failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector93TurnConformanceReplayToolSuccess); - failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector94TurnConformanceRunAssistantToolCallsMetadata); - failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector95TurnConformanceRunAsyncToolFunction); - failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector96TurnConformanceRunBindingsInjected); - failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector97TurnConformanceRunCancellationBeforeLlm); - failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector98TurnConformanceRunCancellationBetweenIterations); - failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector99TurnConformanceRunCancellationBetweenTools); - failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector100TurnConformanceRunContextNoTrimWhenFits); - failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector101TurnConformanceRunContextPreservesSystemMessages); - failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector102TurnConformanceRunContextTrimBasic); - failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector103TurnConformanceRunEmptyToolResult); - failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector104TurnConformanceRunEventsBasicToolLoop); - failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector105TurnConformanceRunEventsErrorLogged); - failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector106TurnConformanceRunEventsNoTools); - failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector107TurnConformanceRunGuardrailAllPass); - failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector108TurnConformanceRunGuardrailInputDeny); - failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector109TurnConformanceRunGuardrailOutputDeny); - failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector110TurnConformanceRunGuardrailToolDeny); - failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector111TurnConformanceRunMaxIterationsExceeded); - failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector112TurnConformanceRunMultiTurnToolCalls); - failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector113TurnConformanceRunMultipleToolCallsSingleTurn); - failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector114TurnConformanceRunNoToolCalls); - failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector115TurnConformanceRunParallelToolsBasic); - failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector116TurnConformanceRunParallelToolsWithGuardrailDeny); - failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector117TurnConformanceRunParallelToolsWithToolError); - failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector118TurnConformanceRunSingleToolCall); - failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector119TurnConformanceRunSteeringInjectMessage); - failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector120TurnConformanceRunSteeringMultipleMessages); - failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector121TurnConformanceRunToolNotRegisteredError); - failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector122TurnConformanceRunToolResultMessageFormat); - failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector123TurnConformanceRunTurnCancelBeforeContext); - failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector124TurnConformanceRunTurnDelegatedProviderState); - failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector125TurnConformanceRunTurnFinalOutput); - failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector126TurnConformanceRunTurnMemoryContextIsModelVisible); - failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector127TurnConformanceRunTurnOrderedToolRound); - failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector128TurnConformanceRunTurnPermissionDenialIsModelVisible); - failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector129WireConformanceToRequestAnthropicImageFormat); - failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector130WireConformanceToRequestAnthropicMaxTokensRequired); - failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector131WireConformanceToRequestAnthropicOptions); - failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector132WireConformanceToRequestAnthropicSystemSeparate); - failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector133WireConformanceToRequestAnthropicToolWire); - failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector134WireConformanceToRequestChatAudioMp3); - failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector135WireConformanceToRequestChatAudioPart); - failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector136WireConformanceToRequestChatImageBase64); - failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector137WireConformanceToRequestChatImagePart); - failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector138WireConformanceToRequestChatMultipartContent); - failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector139WireConformanceToRequestChatSimple); - failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector140WireConformanceToRequestChatSingleTextOptimized); - failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector141WireConformanceToRequestChatWithOptions); - failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector142WireConformanceToRequestEmbeddingWire); - failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector143WireConformanceToRequestImageWire); - failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector144WireConformanceToRequestKindToJsonTypeMapping); - failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector145WireConformanceToRequestOptionsAdditionalProperties); - failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector146WireConformanceToRequestOptionsMaxCompletionTokens); - failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector147WireConformanceToRequestOptionsStopSequences); - failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector148WireConformanceToRequestResponsesSimple); - failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector149WireConformanceToRequestResponsesStructuredOutput); - failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector150WireConformanceToRequestResponsesWithTools); - failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector151WireConformanceToRequestStructuredOutput); - failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector152WireConformanceToRequestStructuredOutputNestedOptional); - failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector153WireConformanceToRequestToolsBindingsStripped); - failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector154WireConformanceToRequestToolsFunctionWire); - failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector155WireConformanceToRequestToolsNullWhenEmpty); - failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector156WireConformanceToRequestToolsStrictMode); + failed += runCase("LoadConformance.load:untrusted_endpoint_oauth_literal_secret_rejected", VectorConformanceTests::vector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_percent_encoded_host_rejected", VectorConformanceTests::vector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_port_mismatch_rejected", VectorConformanceTests::vector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_query_before_at_rejected", VectorConformanceTests::vector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_scheme_mismatch_rejected", VectorConformanceTests::vector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_lookalike_rejected", VectorConformanceTests::vector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", VectorConformanceTests::vector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_trailing_dot_rejected", VectorConformanceTests::vector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_unicode_homoglyph_rejected", VectorConformanceTests::vector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_userinfo_lookalike_rejected", VectorConformanceTests::vector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", VectorConformanceTests::vector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_boundary_rejected", VectorConformanceTests::vector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected); + failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector78MemoryConformanceOperateClearByCategoryAndAll); + failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector79MemoryConformanceOperateFormatCoreAndRecallResults); + failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector80MemoryConformanceOperateRecallEmptyQueryAndLimit); + failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector81MemoryConformanceOperateRecallRanksByWeightedScore); + failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); + failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector83MemoryConformanceOperateRecallTagsAndCoreBoost); + failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); + failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector85MemoryConformanceOperateRememberDedupsCoreByTags); + failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); + failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector87MemoryConformanceOperateRememberEvictsArchivalFirst); + failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector88MemoryConformanceOperateSnapshotPortRoundTrip); + failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector89MemoryConformanceOperateUpdateIndexOutOfRangeError); + failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector90TurnConformanceReplayMaxIterations); + failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector91TurnConformanceReplayNoTool); + failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector92TurnConformanceReplayPermissionDenied); + failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector93TurnConformanceReplayToolFailure); + failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector94TurnConformanceReplayToolSuccess); + failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector95TurnConformanceRunAssistantToolCallsMetadata); + failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector96TurnConformanceRunAsyncToolFunction); + failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector97TurnConformanceRunBindingsInjected); + failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector98TurnConformanceRunCancellationBeforeLlm); + failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector99TurnConformanceRunCancellationBetweenIterations); + failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector100TurnConformanceRunCancellationBetweenTools); + failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector101TurnConformanceRunContextNoTrimWhenFits); + failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector102TurnConformanceRunContextPreservesSystemMessages); + failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector103TurnConformanceRunContextTrimBasic); + failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector104TurnConformanceRunEmptyToolResult); + failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector105TurnConformanceRunEventsBasicToolLoop); + failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector106TurnConformanceRunEventsErrorLogged); + failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector107TurnConformanceRunEventsNoTools); + failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector108TurnConformanceRunGuardrailAllPass); + failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector109TurnConformanceRunGuardrailInputDeny); + failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector110TurnConformanceRunGuardrailOutputDeny); + failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector111TurnConformanceRunGuardrailToolDeny); + failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector112TurnConformanceRunMaxIterationsExceeded); + failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector113TurnConformanceRunMultiTurnToolCalls); + failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector114TurnConformanceRunMultipleToolCallsSingleTurn); + failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector115TurnConformanceRunNoToolCalls); + failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector116TurnConformanceRunParallelToolsBasic); + failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector117TurnConformanceRunParallelToolsWithGuardrailDeny); + failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector118TurnConformanceRunParallelToolsWithToolError); + failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector119TurnConformanceRunSingleToolCall); + failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector120TurnConformanceRunSteeringInjectMessage); + failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector121TurnConformanceRunSteeringMultipleMessages); + failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector122TurnConformanceRunToolNotRegisteredError); + failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector123TurnConformanceRunToolResultMessageFormat); + failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector124TurnConformanceRunTurnCancelBeforeContext); + failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector125TurnConformanceRunTurnDelegatedProviderState); + failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector126TurnConformanceRunTurnFinalOutput); + failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector127TurnConformanceRunTurnMemoryContextIsModelVisible); + failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector128TurnConformanceRunTurnOrderedToolRound); + failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector129TurnConformanceRunTurnPermissionDenialIsModelVisible); + failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector130WireConformanceToRequestAnthropicImageFormat); + failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector131WireConformanceToRequestAnthropicMaxTokensRequired); + failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector132WireConformanceToRequestAnthropicOptions); + failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector133WireConformanceToRequestAnthropicSystemSeparate); + failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector134WireConformanceToRequestAnthropicToolWire); + failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector135WireConformanceToRequestChatAudioMp3); + failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector136WireConformanceToRequestChatAudioPart); + failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector137WireConformanceToRequestChatImageBase64); + failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector138WireConformanceToRequestChatImagePart); + failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector139WireConformanceToRequestChatMultipartContent); + failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector140WireConformanceToRequestChatSimple); + failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector141WireConformanceToRequestChatSingleTextOptimized); + failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector142WireConformanceToRequestChatWithOptions); + failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector143WireConformanceToRequestEmbeddingWire); + failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector144WireConformanceToRequestImageWire); + failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector145WireConformanceToRequestKindToJsonTypeMapping); + failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector146WireConformanceToRequestOptionsAdditionalProperties); + failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector147WireConformanceToRequestOptionsMaxCompletionTokens); + failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector148WireConformanceToRequestOptionsStopSequences); + failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector149WireConformanceToRequestResponsesSimple); + failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector150WireConformanceToRequestResponsesStructuredOutput); + failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector151WireConformanceToRequestResponsesWithTools); + failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector152WireConformanceToRequestStructuredOutput); + failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector153WireConformanceToRequestStructuredOutputNestedOptional); + failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector154WireConformanceToRequestToolsBindingsStripped); + failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector155WireConformanceToRequestToolsFunctionWire); + failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector156WireConformanceToRequestToolsNullWhenEmpty); + failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector157WireConformanceToRequestToolsStrictMode); if (failed > 0) { throw new AssertionError(failed + " @vector conformance failure(s)"); } diff --git a/runtime/python/prompty/README.md b/runtime/python/prompty/README.md index 4d4e40a7e..2dc20f5c5 100644 --- a/runtime/python/prompty/README.md +++ b/runtime/python/prompty/README.md @@ -296,7 +296,7 @@ input value. ### Endpoint Trust -A `.prompty` file can't send ambient credentials (`DefaultAzureCredential` tokens, `OPENAI_API_KEY` and similar env keys) to an endpoint it picked itself. Unless the connection uses a literal `apiKey` (`kind: key`) or `clientSecret` (`kind: oauth`), a literal `endpoint`/`tokenUrl` must be trusted, or `load()` raises `PromptyLoadError("untrusted_endpoint")`. +A `.prompty` file can't send ambient credentials (`DefaultAzureCredential` tokens, `OPENAI_API_KEY` and similar env keys) to an endpoint it picked itself. Unless the connection uses a literal `apiKey` (`kind: key`), a literal `endpoint`/`tokenUrl` must be trusted, or `load()` raises `PromptyLoadError("untrusted_endpoint")`. - Built-in trusted origins: `api.openai.com`, `api.anthropic.com`, `*.openai.azure.com`, `*.services.ai.azure.com`, `*.cognitiveservices.azure.com`, `login.microsoftonline.com` (all `https`). - Supply the endpoint from the host: `endpoint: ${env:MY_ENDPOINT}`. diff --git a/runtime/python/prompty/prompty/core/endpoint_trust.py b/runtime/python/prompty/prompty/core/endpoint_trust.py index dc2dd235a..d57549ed8 100644 --- a/runtime/python/prompty/prompty/core/endpoint_trust.py +++ b/runtime/python/prompty/prompty/core/endpoint_trust.py @@ -29,7 +29,7 @@ ) _DEFAULT_PORTS = {"http": 80, "https": 443} -_CREDENTIAL_KEYS: dict[str, str] = {"key": "apiKey", "oauth": "clientSecret"} +_CREDENTIAL_KEYS: dict[str, str] = {"key": "apiKey"} _DESTINATION_KEYS = ("endpoint", "tokenUrl") diff --git a/runtime/python/prompty/prompty/model/vector_conformance.py b/runtime/python/prompty/prompty/model/vector_conformance.py index 1d4cf7de8..f0505d823 100644 --- a/runtime/python/prompty/prompty/model/vector_conformance.py +++ b/runtime/python/prompty/prompty/model/vector_conformance.py @@ -635,6 +635,16 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: except Exception as error: # noqa: BLE001 caught = error assert caught is not None, "untrusted_endpoint_no_key_rejected: expected an error" + # vector: untrusted_endpoint_oauth_literal_secret_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_oauth_literal_secret_rejected: expected an error" # vector: untrusted_endpoint_percent_encoded_host_rejected source = json.loads("null") caught = None diff --git a/runtime/python/prompty/tests/model/test_vector_conformance.py b/runtime/python/prompty/tests/model/test_vector_conformance.py index 79a1b18de..0f41aebdc 100644 --- a/runtime/python/prompty/tests/model/test_vector_conformance.py +++ b/runtime/python/prompty/tests/model/test_vector_conformance.py @@ -2266,7 +2266,39 @@ async def test_vector_65_loadconformance_load_untrusted_endpoint_no_key_rejected await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_66_loadconformance_load_untrusted_endpoint_percent_encoded_host_rejected(): +async def test_vector_66_loadconformance_load_untrusted_endpoint_oauth_literal_secret_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_oauth_literal_secret_rejected", + "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-oauth-literal", + "model": { + "id": "gpt-4", + "provider": "anthropic", + "connection": { + "kind": "oauth", + "endpoint": "https://attacker.example.com", + "clientId": "decoy", + "clientSecret": "decoy", + "tokenUrl": "https://login.microsoftonline.com/t/oauth2/v2.0/token" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_67_loadconformance_load_untrusted_endpoint_percent_encoded_host_rejected(): vector_json = r""" { "name": "untrusted_endpoint_percent_encoded_host_rejected", @@ -2294,7 +2326,7 @@ async def test_vector_66_loadconformance_load_untrusted_endpoint_percent_encoded await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_67_loadconformance_load_untrusted_endpoint_port_mismatch_rejected(): +async def test_vector_68_loadconformance_load_untrusted_endpoint_port_mismatch_rejected(): vector_json = r""" { "name": "untrusted_endpoint_port_mismatch_rejected", @@ -2322,7 +2354,7 @@ async def test_vector_67_loadconformance_load_untrusted_endpoint_port_mismatch_r await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_68_loadconformance_load_untrusted_endpoint_query_before_at_rejected(): +async def test_vector_69_loadconformance_load_untrusted_endpoint_query_before_at_rejected(): vector_json = r""" { "name": "untrusted_endpoint_query_before_at_rejected", @@ -2350,7 +2382,7 @@ async def test_vector_68_loadconformance_load_untrusted_endpoint_query_before_at await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_69_loadconformance_load_untrusted_endpoint_scheme_mismatch_rejected(): +async def test_vector_70_loadconformance_load_untrusted_endpoint_scheme_mismatch_rejected(): vector_json = r""" { "name": "untrusted_endpoint_scheme_mismatch_rejected", @@ -2378,7 +2410,7 @@ async def test_vector_69_loadconformance_load_untrusted_endpoint_scheme_mismatch await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_70_loadconformance_load_untrusted_endpoint_suffix_lookalike_rejected(): +async def test_vector_71_loadconformance_load_untrusted_endpoint_suffix_lookalike_rejected(): vector_json = r""" { "name": "untrusted_endpoint_suffix_lookalike_rejected", @@ -2406,7 +2438,7 @@ async def test_vector_70_loadconformance_load_untrusted_endpoint_suffix_lookalik await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_71_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected(): +async def test_vector_72_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected(): vector_json = r""" { "name": "untrusted_endpoint_suffix_spoof_rejected", @@ -2437,7 +2469,7 @@ async def test_vector_71_loadconformance_load_untrusted_endpoint_suffix_spoof_re await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_72_loadconformance_load_untrusted_endpoint_trailing_dot_rejected(): +async def test_vector_73_loadconformance_load_untrusted_endpoint_trailing_dot_rejected(): vector_json = r""" { "name": "untrusted_endpoint_trailing_dot_rejected", @@ -2465,7 +2497,7 @@ async def test_vector_72_loadconformance_load_untrusted_endpoint_trailing_dot_re await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_73_loadconformance_load_untrusted_endpoint_unicode_homoglyph_rejected(): +async def test_vector_74_loadconformance_load_untrusted_endpoint_unicode_homoglyph_rejected(): vector_json = r""" { "name": "untrusted_endpoint_unicode_homoglyph_rejected", @@ -2493,7 +2525,7 @@ async def test_vector_73_loadconformance_load_untrusted_endpoint_unicode_homogly await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_74_loadconformance_load_untrusted_endpoint_userinfo_lookalike_rejected(): +async def test_vector_75_loadconformance_load_untrusted_endpoint_userinfo_lookalike_rejected(): vector_json = r""" { "name": "untrusted_endpoint_userinfo_lookalike_rejected", @@ -2521,7 +2553,7 @@ async def test_vector_74_loadconformance_load_untrusted_endpoint_userinfo_lookal await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_75_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected(): +async def test_vector_76_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected(): vector_json = r""" { "name": "untrusted_endpoint_wildcard_apex_rejected", @@ -2549,7 +2581,7 @@ async def test_vector_75_loadconformance_load_untrusted_endpoint_wildcard_apex_r await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_76_loadconformance_load_untrusted_endpoint_wildcard_boundary_rejected(): +async def test_vector_77_loadconformance_load_untrusted_endpoint_wildcard_boundary_rejected(): vector_json = r""" { "name": "untrusted_endpoint_wildcard_boundary_rejected", @@ -2577,7 +2609,7 @@ async def test_vector_76_loadconformance_load_untrusted_endpoint_wildcard_bounda await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_77_memoryconformance_operate_clear_by_category_and_all(): +async def test_vector_78_memoryconformance_operate_clear_by_category_and_all(): vector_json = r""" { "name": "clear_by_category_and_all", @@ -2629,7 +2661,7 @@ async def test_vector_77_memoryconformance_operate_clear_by_category_and_all(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_78_memoryconformance_operate_format_core_and_recall_results(): +async def test_vector_79_memoryconformance_operate_format_core_and_recall_results(): vector_json = r""" { "name": "format_core_and_recall_results", @@ -2669,7 +2701,7 @@ async def test_vector_78_memoryconformance_operate_format_core_and_recall_result await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_79_memoryconformance_operate_recall_empty_query_and_limit(): +async def test_vector_80_memoryconformance_operate_recall_empty_query_and_limit(): vector_json = r""" { "name": "recall_empty_query_and_limit", @@ -2722,7 +2754,7 @@ async def test_vector_79_memoryconformance_operate_recall_empty_query_and_limit( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_80_memoryconformance_operate_recall_ranks_by_weighted_score(): +async def test_vector_81_memoryconformance_operate_recall_ranks_by_weighted_score(): vector_json = r""" { "name": "recall_ranks_by_weighted_score", @@ -2779,7 +2811,7 @@ async def test_vector_80_memoryconformance_operate_recall_ranks_by_weighted_scor await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_81_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): +async def test_vector_82_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): vector_json = r""" { "name": "recall_score_ties_preserve_insertion_order", @@ -2835,7 +2867,7 @@ async def test_vector_81_memoryconformance_operate_recall_score_ties_preserve_in await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_82_memoryconformance_operate_recall_tags_and_core_boost(): +async def test_vector_83_memoryconformance_operate_recall_tags_and_core_boost(): vector_json = r""" { "name": "recall_tags_and_core_boost", @@ -2894,7 +2926,7 @@ async def test_vector_82_memoryconformance_operate_recall_tags_and_core_boost(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_83_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): +async def test_vector_84_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): vector_json = r""" { "name": "recall_tokenization_punctuation_unicode_and_dedup", @@ -2953,7 +2985,7 @@ async def test_vector_83_memoryconformance_operate_recall_tokenization_punctuati await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_84_memoryconformance_operate_remember_dedups_core_by_tags(): +async def test_vector_85_memoryconformance_operate_remember_dedups_core_by_tags(): vector_json = r""" { "name": "remember_dedups_core_by_tags", @@ -3015,7 +3047,7 @@ async def test_vector_84_memoryconformance_operate_remember_dedups_core_by_tags( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_85_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): +async def test_vector_86_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): vector_json = r""" { "name": "remember_dedups_core_missing_and_empty_tags", @@ -3069,7 +3101,7 @@ async def test_vector_85_memoryconformance_operate_remember_dedups_core_missing_ await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_86_memoryconformance_operate_remember_evicts_archival_first(): +async def test_vector_87_memoryconformance_operate_remember_evicts_archival_first(): vector_json = r""" { "name": "remember_evicts_archival_first", @@ -3128,7 +3160,7 @@ async def test_vector_86_memoryconformance_operate_remember_evicts_archival_firs await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_87_memoryconformance_operate_snapshot_port_round_trip(): +async def test_vector_88_memoryconformance_operate_snapshot_port_round_trip(): vector_json = r""" { "name": "snapshot_port_round_trip", @@ -3166,7 +3198,7 @@ async def test_vector_87_memoryconformance_operate_snapshot_port_round_trip(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_88_memoryconformance_operate_update_index_out_of_range_error(): +async def test_vector_89_memoryconformance_operate_update_index_out_of_range_error(): vector_json = r""" { "name": "update_index_out_of_range_error", @@ -3198,7 +3230,7 @@ async def test_vector_88_memoryconformance_operate_update_index_out_of_range_err await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_89_turnconformance_replay_max_iterations(): +async def test_vector_90_turnconformance_replay_max_iterations(): vector_json = r""" { "name": "max_iterations", @@ -3233,7 +3265,7 @@ async def test_vector_89_turnconformance_replay_max_iterations(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_90_turnconformance_replay_no_tool(): +async def test_vector_91_turnconformance_replay_no_tool(): vector_json = r""" { "name": "no_tool", @@ -3264,7 +3296,7 @@ async def test_vector_90_turnconformance_replay_no_tool(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_91_turnconformance_replay_permission_denied(): +async def test_vector_92_turnconformance_replay_permission_denied(): vector_json = r""" { "name": "permission_denied", @@ -3298,7 +3330,7 @@ async def test_vector_91_turnconformance_replay_permission_denied(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_92_turnconformance_replay_tool_failure(): +async def test_vector_93_turnconformance_replay_tool_failure(): vector_json = r""" { "name": "tool_failure", @@ -3334,7 +3366,7 @@ async def test_vector_92_turnconformance_replay_tool_failure(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_93_turnconformance_replay_tool_success(): +async def test_vector_94_turnconformance_replay_tool_success(): vector_json = r""" { "name": "tool_success", @@ -3370,7 +3402,7 @@ async def test_vector_93_turnconformance_replay_tool_success(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_94_turnconformance_run_assistant_tool_calls_metadata(): +async def test_vector_95_turnconformance_run_assistant_tool_calls_metadata(): vector_json = r""" { "name": "assistant_tool_calls_metadata", @@ -3522,7 +3554,7 @@ async def test_vector_94_turnconformance_run_assistant_tool_calls_metadata(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_95_turnconformance_run_async_tool_function(): +async def test_vector_96_turnconformance_run_async_tool_function(): vector_json = r""" { "name": "async_tool_function", @@ -3632,7 +3664,7 @@ async def test_vector_95_turnconformance_run_async_tool_function(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_96_turnconformance_run_bindings_injected(): +async def test_vector_97_turnconformance_run_bindings_injected(): vector_json = r""" { "name": "bindings_injected", @@ -3762,7 +3794,7 @@ async def test_vector_96_turnconformance_run_bindings_injected(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_97_turnconformance_run_cancellation_before_llm(): +async def test_vector_98_turnconformance_run_cancellation_before_llm(): vector_json = r""" { "name": "cancellation_before_llm", @@ -3823,7 +3855,7 @@ async def test_vector_97_turnconformance_run_cancellation_before_llm(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_98_turnconformance_run_cancellation_between_iterations(): +async def test_vector_99_turnconformance_run_cancellation_between_iterations(): vector_json = r""" { "name": "cancellation_between_iterations", @@ -3953,7 +3985,7 @@ async def test_vector_98_turnconformance_run_cancellation_between_iterations(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_99_turnconformance_run_cancellation_between_tools(): +async def test_vector_100_turnconformance_run_cancellation_between_tools(): vector_json = r""" { "name": "cancellation_between_tools", @@ -4092,7 +4124,7 @@ async def test_vector_99_turnconformance_run_cancellation_between_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_100_turnconformance_run_context_no_trim_when_fits(): +async def test_vector_101_turnconformance_run_context_no_trim_when_fits(): vector_json = r""" { "name": "context_no_trim_when_fits", @@ -4163,7 +4195,7 @@ async def test_vector_100_turnconformance_run_context_no_trim_when_fits(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_101_turnconformance_run_context_preserves_system_messages(): +async def test_vector_102_turnconformance_run_context_preserves_system_messages(): vector_json = r""" { "name": "context_preserves_system_messages", @@ -4270,7 +4302,7 @@ async def test_vector_101_turnconformance_run_context_preserves_system_messages( await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_102_turnconformance_run_context_trim_basic(): +async def test_vector_103_turnconformance_run_context_trim_basic(): vector_json = r""" { "name": "context_trim_basic", @@ -4428,7 +4460,7 @@ async def test_vector_102_turnconformance_run_context_trim_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_103_turnconformance_run_empty_tool_result(): +async def test_vector_104_turnconformance_run_empty_tool_result(): vector_json = r""" { "name": "empty_tool_result", @@ -4543,7 +4575,7 @@ async def test_vector_103_turnconformance_run_empty_tool_result(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_104_turnconformance_run_events_basic_tool_loop(): +async def test_vector_105_turnconformance_run_events_basic_tool_loop(): vector_json = r""" { "name": "events_basic_tool_loop", @@ -4689,7 +4721,7 @@ async def test_vector_104_turnconformance_run_events_basic_tool_loop(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_105_turnconformance_run_events_error_logged(): +async def test_vector_106_turnconformance_run_events_error_logged(): vector_json = r""" { "name": "events_error_logged", @@ -4815,7 +4847,7 @@ async def test_vector_105_turnconformance_run_events_error_logged(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_106_turnconformance_run_events_no_tools(): +async def test_vector_107_turnconformance_run_events_no_tools(): vector_json = r""" { "name": "events_no_tools", @@ -4893,7 +4925,7 @@ async def test_vector_106_turnconformance_run_events_no_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_107_turnconformance_run_guardrail_all_pass(): +async def test_vector_108_turnconformance_run_guardrail_all_pass(): vector_json = r""" { "name": "guardrail_all_pass", @@ -5016,7 +5048,7 @@ async def test_vector_107_turnconformance_run_guardrail_all_pass(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_108_turnconformance_run_guardrail_input_deny(): +async def test_vector_109_turnconformance_run_guardrail_input_deny(): vector_json = r""" { "name": "guardrail_input_deny", @@ -5072,7 +5104,7 @@ async def test_vector_108_turnconformance_run_guardrail_input_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_109_turnconformance_run_guardrail_output_deny(): +async def test_vector_110_turnconformance_run_guardrail_output_deny(): vector_json = r""" { "name": "guardrail_output_deny", @@ -5150,7 +5182,7 @@ async def test_vector_109_turnconformance_run_guardrail_output_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_110_turnconformance_run_guardrail_tool_deny(): +async def test_vector_111_turnconformance_run_guardrail_tool_deny(): vector_json = r""" { "name": "guardrail_tool_deny", @@ -5304,7 +5336,7 @@ async def test_vector_110_turnconformance_run_guardrail_tool_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_111_turnconformance_run_max_iterations_exceeded(): +async def test_vector_112_turnconformance_run_max_iterations_exceeded(): vector_json = r""" { "name": "max_iterations_exceeded", @@ -5814,7 +5846,7 @@ async def test_vector_111_turnconformance_run_max_iterations_exceeded(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_112_turnconformance_run_multi_turn_tool_calls(): +async def test_vector_113_turnconformance_run_multi_turn_tool_calls(): vector_json = r""" { "name": "multi_turn_tool_calls", @@ -5991,7 +6023,7 @@ async def test_vector_112_turnconformance_run_multi_turn_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_113_turnconformance_run_multiple_tool_calls_single_turn(): +async def test_vector_114_turnconformance_run_multiple_tool_calls_single_turn(): vector_json = r""" { "name": "multiple_tool_calls_single_turn", @@ -6172,7 +6204,7 @@ async def test_vector_113_turnconformance_run_multiple_tool_calls_single_turn(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_114_turnconformance_run_no_tool_calls(): +async def test_vector_115_turnconformance_run_no_tool_calls(): vector_json = r""" { "name": "no_tool_calls", @@ -6240,7 +6272,7 @@ async def test_vector_114_turnconformance_run_no_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_115_turnconformance_run_parallel_tools_basic(): +async def test_vector_116_turnconformance_run_parallel_tools_basic(): vector_json = r""" { "name": "parallel_tools_basic", @@ -6413,7 +6445,7 @@ async def test_vector_115_turnconformance_run_parallel_tools_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_116_turnconformance_run_parallel_tools_with_guardrail_deny(): +async def test_vector_117_turnconformance_run_parallel_tools_with_guardrail_deny(): vector_json = r""" { "name": "parallel_tools_with_guardrail_deny", @@ -6604,7 +6636,7 @@ async def test_vector_116_turnconformance_run_parallel_tools_with_guardrail_deny await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_117_turnconformance_run_parallel_tools_with_tool_error(): +async def test_vector_118_turnconformance_run_parallel_tools_with_tool_error(): vector_json = r""" { "name": "parallel_tools_with_tool_error", @@ -6777,7 +6809,7 @@ async def test_vector_117_turnconformance_run_parallel_tools_with_tool_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_118_turnconformance_run_single_tool_call(): +async def test_vector_119_turnconformance_run_single_tool_call(): vector_json = r""" { "name": "single_tool_call", @@ -6924,7 +6956,7 @@ async def test_vector_118_turnconformance_run_single_tool_call(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_119_turnconformance_run_steering_inject_message(): +async def test_vector_120_turnconformance_run_steering_inject_message(): vector_json = r""" { "name": "steering_inject_message", @@ -7132,7 +7164,7 @@ async def test_vector_119_turnconformance_run_steering_inject_message(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_120_turnconformance_run_steering_multiple_messages(): +async def test_vector_121_turnconformance_run_steering_multiple_messages(): vector_json = r""" { "name": "steering_multiple_messages", @@ -7302,7 +7334,7 @@ async def test_vector_120_turnconformance_run_steering_multiple_messages(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_121_turnconformance_run_tool_not_registered_error(): +async def test_vector_122_turnconformance_run_tool_not_registered_error(): vector_json = r""" { "name": "tool_not_registered_error", @@ -7386,7 +7418,7 @@ async def test_vector_121_turnconformance_run_tool_not_registered_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_122_turnconformance_run_tool_result_message_format(): +async def test_vector_123_turnconformance_run_tool_result_message_format(): vector_json = r""" { "name": "tool_result_message_format", @@ -7507,7 +7539,7 @@ async def test_vector_122_turnconformance_run_tool_result_message_format(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_123_turnconformance_runturn_cancel_before_context(): +async def test_vector_124_turnconformance_runturn_cancel_before_context(): vector_json = r""" { "name": "cancel_before_context", @@ -7539,7 +7571,7 @@ async def test_vector_123_turnconformance_runturn_cancel_before_context(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_124_turnconformance_runturn_delegated_provider_state(): +async def test_vector_125_turnconformance_runturn_delegated_provider_state(): vector_json = r""" { "name": "delegated_provider_state", @@ -7606,7 +7638,7 @@ async def test_vector_124_turnconformance_runturn_delegated_provider_state(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_125_turnconformance_runturn_final_output(): +async def test_vector_126_turnconformance_runturn_final_output(): vector_json = r""" { "name": "final_output", @@ -7651,7 +7683,7 @@ async def test_vector_125_turnconformance_runturn_final_output(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_126_turnconformance_runturn_memory_context_is_model_visible(): +async def test_vector_127_turnconformance_runturn_memory_context_is_model_visible(): vector_json = r""" { "name": "memory_context_is_model_visible", @@ -7713,7 +7745,7 @@ async def test_vector_126_turnconformance_runturn_memory_context_is_model_visibl await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_127_turnconformance_runturn_ordered_tool_round(): +async def test_vector_128_turnconformance_runturn_ordered_tool_round(): vector_json = r""" { "name": "ordered_tool_round", @@ -7804,7 +7836,7 @@ async def test_vector_127_turnconformance_runturn_ordered_tool_round(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_128_turnconformance_runturn_permission_denial_is_model_visible(): +async def test_vector_129_turnconformance_runturn_permission_denial_is_model_visible(): vector_json = r""" { "name": "permission_denial_is_model_visible", @@ -7855,7 +7887,7 @@ async def test_vector_128_turnconformance_runturn_permission_denial_is_model_vis await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_129_wireconformance_torequest_anthropic_image_format(): +async def test_vector_130_wireconformance_torequest_anthropic_image_format(): vector_json = r""" { "name": "anthropic_image_format", @@ -7925,7 +7957,7 @@ async def test_vector_129_wireconformance_torequest_anthropic_image_format(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_130_wireconformance_torequest_anthropic_max_tokens_required(): +async def test_vector_131_wireconformance_torequest_anthropic_max_tokens_required(): vector_json = r""" { "name": "anthropic_max_tokens_required", @@ -7980,7 +8012,7 @@ async def test_vector_130_wireconformance_torequest_anthropic_max_tokens_require await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_131_wireconformance_torequest_anthropic_options(): +async def test_vector_132_wireconformance_torequest_anthropic_options(): vector_json = r""" { "name": "anthropic_options", @@ -8049,7 +8081,7 @@ async def test_vector_131_wireconformance_torequest_anthropic_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_132_wireconformance_torequest_anthropic_system_separate(): +async def test_vector_133_wireconformance_torequest_anthropic_system_separate(): vector_json = r""" { "name": "anthropic_system_separate", @@ -8116,7 +8148,7 @@ async def test_vector_132_wireconformance_torequest_anthropic_system_separate(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_133_wireconformance_torequest_anthropic_tool_wire(): +async def test_vector_134_wireconformance_torequest_anthropic_tool_wire(): vector_json = r""" { "name": "anthropic_tool_wire", @@ -8201,7 +8233,7 @@ async def test_vector_133_wireconformance_torequest_anthropic_tool_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_134_wireconformance_torequest_chat_audio_mp3(): +async def test_vector_135_wireconformance_torequest_chat_audio_mp3(): vector_json = r""" { "name": "chat_audio_mp3", @@ -8259,7 +8291,7 @@ async def test_vector_134_wireconformance_torequest_chat_audio_mp3(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_135_wireconformance_torequest_chat_audio_part(): +async def test_vector_136_wireconformance_torequest_chat_audio_part(): vector_json = r""" { "name": "chat_audio_part", @@ -8317,7 +8349,7 @@ async def test_vector_135_wireconformance_torequest_chat_audio_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_136_wireconformance_torequest_chat_image_base64(): +async def test_vector_137_wireconformance_torequest_chat_image_base64(): vector_json = r""" { "name": "chat_image_base64", @@ -8381,7 +8413,7 @@ async def test_vector_136_wireconformance_torequest_chat_image_base64(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_137_wireconformance_torequest_chat_image_part(): +async def test_vector_138_wireconformance_torequest_chat_image_part(): vector_json = r""" { "name": "chat_image_part", @@ -8445,7 +8477,7 @@ async def test_vector_137_wireconformance_torequest_chat_image_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_138_wireconformance_torequest_chat_multipart_content(): +async def test_vector_139_wireconformance_torequest_chat_multipart_content(): vector_json = r""" { "name": "chat_multipart_content", @@ -8509,7 +8541,7 @@ async def test_vector_138_wireconformance_torequest_chat_multipart_content(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_139_wireconformance_torequest_chat_simple(): +async def test_vector_140_wireconformance_torequest_chat_simple(): vector_json = r""" { "name": "chat_simple", @@ -8571,7 +8603,7 @@ async def test_vector_139_wireconformance_torequest_chat_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_140_wireconformance_torequest_chat_single_text_optimized(): +async def test_vector_141_wireconformance_torequest_chat_single_text_optimized(): vector_json = r""" { "name": "chat_single_text_optimized", @@ -8620,7 +8652,7 @@ async def test_vector_140_wireconformance_torequest_chat_single_text_optimized() await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_141_wireconformance_torequest_chat_with_options(): +async def test_vector_142_wireconformance_torequest_chat_with_options(): vector_json = r""" { "name": "chat_with_options", @@ -8688,7 +8720,7 @@ async def test_vector_141_wireconformance_torequest_chat_with_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_142_wireconformance_torequest_embedding_wire(): +async def test_vector_143_wireconformance_torequest_embedding_wire(): vector_json = r""" { "name": "embedding_wire", @@ -8732,7 +8764,7 @@ async def test_vector_142_wireconformance_torequest_embedding_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_143_wireconformance_torequest_image_wire(): +async def test_vector_144_wireconformance_torequest_image_wire(): vector_json = r""" { "name": "image_wire", @@ -8776,7 +8808,7 @@ async def test_vector_143_wireconformance_torequest_image_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_144_wireconformance_torequest_kind_to_json_type_mapping(): +async def test_vector_145_wireconformance_torequest_kind_to_json_type_mapping(): vector_json = r""" { "name": "kind_to_json_type_mapping", @@ -8895,7 +8927,7 @@ async def test_vector_144_wireconformance_torequest_kind_to_json_type_mapping(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_145_wireconformance_torequest_options_additional_properties(): +async def test_vector_146_wireconformance_torequest_options_additional_properties(): vector_json = r""" { "name": "options_additional_properties", @@ -8949,7 +8981,7 @@ async def test_vector_145_wireconformance_torequest_options_additional_propertie await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_146_wireconformance_torequest_options_max_completion_tokens(): +async def test_vector_147_wireconformance_torequest_options_max_completion_tokens(): vector_json = r""" { "name": "options_max_completion_tokens", @@ -9001,7 +9033,7 @@ async def test_vector_146_wireconformance_torequest_options_max_completion_token await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_147_wireconformance_torequest_options_stop_sequences(): +async def test_vector_148_wireconformance_torequest_options_stop_sequences(): vector_json = r""" { "name": "options_stop_sequences", @@ -9059,7 +9091,7 @@ async def test_vector_147_wireconformance_torequest_options_stop_sequences(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_148_wireconformance_torequest_responses_simple(): +async def test_vector_149_wireconformance_torequest_responses_simple(): vector_json = r""" { "name": "responses_simple", @@ -9118,7 +9150,7 @@ async def test_vector_148_wireconformance_torequest_responses_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_149_wireconformance_torequest_responses_structured_output(): +async def test_vector_150_wireconformance_torequest_responses_structured_output(): vector_json = r""" { "name": "responses_structured_output", @@ -9205,7 +9237,7 @@ async def test_vector_149_wireconformance_torequest_responses_structured_output( await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_150_wireconformance_torequest_responses_with_tools(): +async def test_vector_151_wireconformance_torequest_responses_with_tools(): vector_json = r""" { "name": "responses_with_tools", @@ -9285,7 +9317,7 @@ async def test_vector_150_wireconformance_torequest_responses_with_tools(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_151_wireconformance_torequest_structured_output(): +async def test_vector_152_wireconformance_torequest_structured_output(): vector_json = r""" { "name": "structured_output", @@ -9368,7 +9400,7 @@ async def test_vector_151_wireconformance_torequest_structured_output(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_152_wireconformance_torequest_structured_output_nested_optional(): +async def test_vector_153_wireconformance_torequest_structured_output_nested_optional(): vector_json = r""" { "name": "structured_output_nested_optional", @@ -9470,7 +9502,7 @@ async def test_vector_152_wireconformance_torequest_structured_output_nested_opt await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_153_wireconformance_torequest_tools_bindings_stripped(): +async def test_vector_154_wireconformance_torequest_tools_bindings_stripped(): vector_json = r""" { "name": "tools_bindings_stripped", @@ -9562,7 +9594,7 @@ async def test_vector_153_wireconformance_torequest_tools_bindings_stripped(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_154_wireconformance_torequest_tools_function_wire(): +async def test_vector_155_wireconformance_torequest_tools_function_wire(): vector_json = r""" { "name": "tools_function_wire", @@ -9644,7 +9676,7 @@ async def test_vector_154_wireconformance_torequest_tools_function_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_155_wireconformance_torequest_tools_null_when_empty(): +async def test_vector_156_wireconformance_torequest_tools_null_when_empty(): vector_json = r""" { "name": "tools_null_when_empty", @@ -9693,7 +9725,7 @@ async def test_vector_155_wireconformance_torequest_tools_null_when_empty(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_156_wireconformance_torequest_tools_strict_mode(): +async def test_vector_157_wireconformance_torequest_tools_strict_mode(): vector_json = r""" { "name": "tools_strict_mode", diff --git a/runtime/rust/prompty/README.md b/runtime/rust/prompty/README.md index ae71736af..7e3266792 100644 --- a/runtime/rust/prompty/README.md +++ b/runtime/rust/prompty/README.md @@ -48,7 +48,7 @@ guardrail rewrite and before the durable success commit; rejection records a fai - `trusted_endpoints` — extra origins allowed to receive ambient credentials. A `.prompty` file is untrusted. When its connection has no literal file-supplied -credential (`apiKey` for `kind: key`, `clientSecret` for `kind: oauth`), executors fall +credential (a literal `apiKey` on `kind: key`), executors fall back to ambient credentials (Entra ID tokens, `*_API_KEY` env vars). Loading fails with `LoadError::UntrustedEndpoint` if such a connection's `endpoint` / `tokenUrl` is chosen by the file (a literal, `${file:...}`, or an `${env:VAR:default}` default) and its origin diff --git a/runtime/rust/prompty/src/loader/endpoint_trust.rs b/runtime/rust/prompty/src/loader/endpoint_trust.rs index 5f12c4ec2..c0b3740ef 100644 --- a/runtime/rust/prompty/src/loader/endpoint_trust.rs +++ b/runtime/rust/prompty/src/loader/endpoint_trust.rs @@ -137,7 +137,7 @@ fn file_supplied_credential(connection: &serde_json::Map) -> bool // Only the credential field the declared kind actually consumes counts; any other kind is ambient. let key = match connection.get("kind").and_then(Value::as_str) { Some("key") => "apiKey", - Some("oauth") => "clientSecret", + _ => return false, }; matches!(connection.get(key), Some(Value::String(s)) if !s.is_empty() && !is_reference(s)) diff --git a/runtime/rust/prompty/src/model/vector_conformance.rs b/runtime/rust/prompty/src/model/vector_conformance.rs index 6c80c5106..d8ee20e91 100644 --- a/runtime/rust/prompty/src/model/vector_conformance.rs +++ b/runtime/rust/prompty/src/model/vector_conformance.rs @@ -1743,6 +1743,20 @@ pub async fn run_load_conformance_conformance(_ seam: S) async t throw SeamConformanceError("untrusted_endpoint_no_key_rejected: expected an error") } } + // vector: untrusted_endpoint_oauth_literal_secret_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_oauth_literal_secret_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_oauth_literal_secret_rejected: expected an error") + } + } // vector: untrusted_endpoint_percent_encoded_host_rejected do { let inputData = "{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}}".data(using: .utf8)! diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift index 9277913b1..cf03038ca 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift @@ -694,7 +694,17 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector66LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() async throws { + func testVector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_oauth_literal_secret_rejected\",\"description\":\"kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_percent_encoded_host_rejected\",\"description\":\"Hosts are compared without percent-decoding, so an encoded host never matches\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -704,7 +714,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector67LoadConformanceLoadUntrustedEndpointPortMismatchRejected() async throws { + func testVector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_port_mismatch_rejected\",\"description\":\"A non-default port is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -714,7 +724,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector68LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() async throws { + func testVector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_query_before_at_rejected\",\"description\":\"A '?' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -724,7 +734,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector69LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() async throws { + func testVector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_scheme_mismatch_rejected\",\"description\":\"Built-in origins are https; http on the same host is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -734,7 +744,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector70LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() async throws { + func testVector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_suffix_lookalike_rejected\",\"description\":\"A trusted host as a label prefix of another domain is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -744,7 +754,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector71LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() async throws { + func testVector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -754,7 +764,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector72LoadConformanceLoadUntrustedEndpointTrailingDotRejected() async throws { + func testVector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_trailing_dot_rejected\",\"description\":\"Hosts are compared exactly; a trailing-dot host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -764,7 +774,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector73LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() async throws { + func testVector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_unicode_homoglyph_rejected\",\"description\":\"A non-ASCII lookalike host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\u{430}pi.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -774,7 +784,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector74LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() async throws { + func testVector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_userinfo_lookalike_rejected\",\"description\":\"A trusted host in userinfo is not the destination host\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -784,7 +794,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector75LoadConformanceLoadUntrustedEndpointWildcardApexRejected() async throws { + func testVector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -794,7 +804,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector76LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() async throws { + func testVector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_wildcard_boundary_rejected\",\"description\":\"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -804,7 +814,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector77MemoryConformanceOperateClearByCategoryAndAll() async throws { + func testVector78MemoryConformanceOperateClearByCategoryAndAll() async throws { guard let vectorData = "{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -814,7 +824,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector78MemoryConformanceOperateFormatCoreAndRecallResults() async throws { + func testVector79MemoryConformanceOperateFormatCoreAndRecallResults() async throws { guard let vectorData = "{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -824,7 +834,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector79MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { + func testVector80MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { guard let vectorData = "{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -834,7 +844,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector80MemoryConformanceOperateRecallRanksByWeightedScore() async throws { + func testVector81MemoryConformanceOperateRecallRanksByWeightedScore() async throws { guard let vectorData = "{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -844,7 +854,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector81MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { + func testVector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { guard let vectorData = "{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -854,7 +864,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector82MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { + func testVector83MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { guard let vectorData = "{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -864,7 +874,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector83MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { + func testVector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { guard let vectorData = "{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\u{e9}\"]},{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\u{e9}\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -874,7 +884,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector84MemoryConformanceOperateRememberDedupsCoreByTags() async throws { + func testVector85MemoryConformanceOperateRememberDedupsCoreByTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -884,7 +894,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector85MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { + func testVector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -894,7 +904,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector86MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { + func testVector87MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { guard let vectorData = "{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -904,7 +914,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector87MemoryConformanceOperateSnapshotPortRoundTrip() async throws { + func testVector88MemoryConformanceOperateSnapshotPortRoundTrip() async throws { guard let vectorData = "{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -914,7 +924,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector88MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { + func testVector89MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { guard let vectorData = "{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -924,7 +934,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector89TurnConformanceReplayMaxIterations() async throws { + func testVector90TurnConformanceReplayMaxIterations() async throws { guard let vectorData = "{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -934,7 +944,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector90TurnConformanceReplayNoTool() async throws { + func testVector91TurnConformanceReplayNoTool() async throws { guard let vectorData = "{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -944,7 +954,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector91TurnConformanceReplayPermissionDenied() async throws { + func testVector92TurnConformanceReplayPermissionDenied() async throws { guard let vectorData = "{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -954,7 +964,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector92TurnConformanceReplayToolFailure() async throws { + func testVector93TurnConformanceReplayToolFailure() async throws { guard let vectorData = "{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -964,7 +974,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector93TurnConformanceReplayToolSuccess() async throws { + func testVector94TurnConformanceReplayToolSuccess() async throws { guard let vectorData = "{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -974,7 +984,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector94TurnConformanceRunAssistantToolCallsMetadata() async throws { + func testVector95TurnConformanceRunAssistantToolCallsMetadata() async throws { guard let vectorData = "{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -984,7 +994,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector95TurnConformanceRunAsyncToolFunction() async throws { + func testVector96TurnConformanceRunAsyncToolFunction() async throws { guard let vectorData = "{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -994,7 +1004,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector96TurnConformanceRunBindingsInjected() async throws { + func testVector97TurnConformanceRunBindingsInjected() async throws { guard let vectorData = "{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1004,7 +1014,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector97TurnConformanceRunCancellationBeforeLlm() async throws { + func testVector98TurnConformanceRunCancellationBeforeLlm() async throws { guard let vectorData = "{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1014,7 +1024,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector98TurnConformanceRunCancellationBetweenIterations() async throws { + func testVector99TurnConformanceRunCancellationBetweenIterations() async throws { guard let vectorData = "{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1024,7 +1034,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector99TurnConformanceRunCancellationBetweenTools() async throws { + func testVector100TurnConformanceRunCancellationBetweenTools() async throws { guard let vectorData = "{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1034,7 +1044,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector100TurnConformanceRunContextNoTrimWhenFits() async throws { + func testVector101TurnConformanceRunContextNoTrimWhenFits() async throws { guard let vectorData = "{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1044,7 +1054,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector101TurnConformanceRunContextPreservesSystemMessages() async throws { + func testVector102TurnConformanceRunContextPreservesSystemMessages() async throws { guard let vectorData = "{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1054,7 +1064,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector102TurnConformanceRunContextTrimBasic() async throws { + func testVector103TurnConformanceRunContextTrimBasic() async throws { guard let vectorData = "{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1064,7 +1074,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector103TurnConformanceRunEmptyToolResult() async throws { + func testVector104TurnConformanceRunEmptyToolResult() async throws { guard let vectorData = "{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1074,7 +1084,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector104TurnConformanceRunEventsBasicToolLoop() async throws { + func testVector105TurnConformanceRunEventsBasicToolLoop() async throws { guard let vectorData = "{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1084,7 +1094,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector105TurnConformanceRunEventsErrorLogged() async throws { + func testVector106TurnConformanceRunEventsErrorLogged() async throws { guard let vectorData = "{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1094,7 +1104,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector106TurnConformanceRunEventsNoTools() async throws { + func testVector107TurnConformanceRunEventsNoTools() async throws { guard let vectorData = "{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1104,7 +1114,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector107TurnConformanceRunGuardrailAllPass() async throws { + func testVector108TurnConformanceRunGuardrailAllPass() async throws { guard let vectorData = "{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1114,7 +1124,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector108TurnConformanceRunGuardrailInputDeny() async throws { + func testVector109TurnConformanceRunGuardrailInputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1124,7 +1134,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector109TurnConformanceRunGuardrailOutputDeny() async throws { + func testVector110TurnConformanceRunGuardrailOutputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1134,7 +1144,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector110TurnConformanceRunGuardrailToolDeny() async throws { + func testVector111TurnConformanceRunGuardrailToolDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1144,7 +1154,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector111TurnConformanceRunMaxIterationsExceeded() async throws { + func testVector112TurnConformanceRunMaxIterationsExceeded() async throws { guard let vectorData = "{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1154,7 +1164,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector112TurnConformanceRunMultiTurnToolCalls() async throws { + func testVector113TurnConformanceRunMultiTurnToolCalls() async throws { guard let vectorData = "{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1164,7 +1174,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector113TurnConformanceRunMultipleToolCallsSingleTurn() async throws { + func testVector114TurnConformanceRunMultipleToolCallsSingleTurn() async throws { guard let vectorData = "{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1174,7 +1184,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector114TurnConformanceRunNoToolCalls() async throws { + func testVector115TurnConformanceRunNoToolCalls() async throws { guard let vectorData = "{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1184,7 +1194,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector115TurnConformanceRunParallelToolsBasic() async throws { + func testVector116TurnConformanceRunParallelToolsBasic() async throws { guard let vectorData = "{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1194,7 +1204,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector116TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { + func testVector117TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1204,7 +1214,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector117TurnConformanceRunParallelToolsWithToolError() async throws { + func testVector118TurnConformanceRunParallelToolsWithToolError() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\u{a7}13.6 Parallel Tools + tool error \u{2014} 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1214,7 +1224,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector118TurnConformanceRunSingleToolCall() async throws { + func testVector119TurnConformanceRunSingleToolCall() async throws { guard let vectorData = "{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1224,7 +1234,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector119TurnConformanceRunSteeringInjectMessage() async throws { + func testVector120TurnConformanceRunSteeringInjectMessage() async throws { guard let vectorData = "{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1234,7 +1244,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector120TurnConformanceRunSteeringMultipleMessages() async throws { + func testVector121TurnConformanceRunSteeringMultipleMessages() async throws { guard let vectorData = "{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1244,7 +1254,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector121TurnConformanceRunToolNotRegisteredError() async throws { + func testVector122TurnConformanceRunToolNotRegisteredError() async throws { guard let vectorData = "{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1254,7 +1264,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector122TurnConformanceRunToolResultMessageFormat() async throws { + func testVector123TurnConformanceRunToolResultMessageFormat() async throws { guard let vectorData = "{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1264,7 +1274,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector123TurnConformanceRunTurnCancelBeforeContext() async throws { + func testVector124TurnConformanceRunTurnCancelBeforeContext() async throws { guard let vectorData = "{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1274,7 +1284,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector124TurnConformanceRunTurnDelegatedProviderState() async throws { + func testVector125TurnConformanceRunTurnDelegatedProviderState() async throws { guard let vectorData = "{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1284,7 +1294,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector125TurnConformanceRunTurnFinalOutput() async throws { + func testVector126TurnConformanceRunTurnFinalOutput() async throws { guard let vectorData = "{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1294,7 +1304,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector126TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { + func testVector127TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { guard let vectorData = "{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1304,7 +1314,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector127TurnConformanceRunTurnOrderedToolRound() async throws { + func testVector128TurnConformanceRunTurnOrderedToolRound() async throws { guard let vectorData = "{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1314,7 +1324,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector128TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { + func testVector129TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { guard let vectorData = "{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1324,7 +1334,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector129WireConformanceToRequestAnthropicImageFormat() async throws { + func testVector130WireConformanceToRequestAnthropicImageFormat() async throws { guard let vectorData = "{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1334,7 +1344,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector130WireConformanceToRequestAnthropicMaxTokensRequired() async throws { + func testVector131WireConformanceToRequestAnthropicMaxTokensRequired() async throws { guard let vectorData = "{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1344,7 +1354,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector131WireConformanceToRequestAnthropicOptions() async throws { + func testVector132WireConformanceToRequestAnthropicOptions() async throws { guard let vectorData = "{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1354,7 +1364,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector132WireConformanceToRequestAnthropicSystemSeparate() async throws { + func testVector133WireConformanceToRequestAnthropicSystemSeparate() async throws { guard let vectorData = "{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1364,7 +1374,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector133WireConformanceToRequestAnthropicToolWire() async throws { + func testVector134WireConformanceToRequestAnthropicToolWire() async throws { guard let vectorData = "{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1374,7 +1384,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector134WireConformanceToRequestChatAudioMp3() async throws { + func testVector135WireConformanceToRequestChatAudioMp3() async throws { guard let vectorData = "{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1384,7 +1394,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector135WireConformanceToRequestChatAudioPart() async throws { + func testVector136WireConformanceToRequestChatAudioPart() async throws { guard let vectorData = "{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1394,7 +1404,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector136WireConformanceToRequestChatImageBase64() async throws { + func testVector137WireConformanceToRequestChatImageBase64() async throws { guard let vectorData = "{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1404,7 +1414,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector137WireConformanceToRequestChatImagePart() async throws { + func testVector138WireConformanceToRequestChatImagePart() async throws { guard let vectorData = "{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1414,7 +1424,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector138WireConformanceToRequestChatMultipartContent() async throws { + func testVector139WireConformanceToRequestChatMultipartContent() async throws { guard let vectorData = "{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1424,7 +1434,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector139WireConformanceToRequestChatSimple() async throws { + func testVector140WireConformanceToRequestChatSimple() async throws { guard let vectorData = "{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1434,7 +1444,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector140WireConformanceToRequestChatSingleTextOptimized() async throws { + func testVector141WireConformanceToRequestChatSingleTextOptimized() async throws { guard let vectorData = "{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1444,7 +1454,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector141WireConformanceToRequestChatWithOptions() async throws { + func testVector142WireConformanceToRequestChatWithOptions() async throws { guard let vectorData = "{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1454,7 +1464,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector142WireConformanceToRequestEmbeddingWire() async throws { + func testVector143WireConformanceToRequestEmbeddingWire() async throws { guard let vectorData = "{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1464,7 +1474,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector143WireConformanceToRequestImageWire() async throws { + func testVector144WireConformanceToRequestImageWire() async throws { guard let vectorData = "{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1474,7 +1484,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector144WireConformanceToRequestKindToJsonTypeMapping() async throws { + func testVector145WireConformanceToRequestKindToJsonTypeMapping() async throws { guard let vectorData = "{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1484,7 +1494,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector145WireConformanceToRequestOptionsAdditionalProperties() async throws { + func testVector146WireConformanceToRequestOptionsAdditionalProperties() async throws { guard let vectorData = "{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1494,7 +1504,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector146WireConformanceToRequestOptionsMaxCompletionTokens() async throws { + func testVector147WireConformanceToRequestOptionsMaxCompletionTokens() async throws { guard let vectorData = "{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1504,7 +1514,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector147WireConformanceToRequestOptionsStopSequences() async throws { + func testVector148WireConformanceToRequestOptionsStopSequences() async throws { guard let vectorData = "{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1514,7 +1524,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector148WireConformanceToRequestResponsesSimple() async throws { + func testVector149WireConformanceToRequestResponsesSimple() async throws { guard let vectorData = "{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1524,7 +1534,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector149WireConformanceToRequestResponsesStructuredOutput() async throws { + func testVector150WireConformanceToRequestResponsesStructuredOutput() async throws { guard let vectorData = "{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1534,7 +1544,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector150WireConformanceToRequestResponsesWithTools() async throws { + func testVector151WireConformanceToRequestResponsesWithTools() async throws { guard let vectorData = "{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1544,7 +1554,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector151WireConformanceToRequestStructuredOutput() async throws { + func testVector152WireConformanceToRequestStructuredOutput() async throws { guard let vectorData = "{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1554,7 +1564,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector152WireConformanceToRequestStructuredOutputNestedOptional() async throws { + func testVector153WireConformanceToRequestStructuredOutputNestedOptional() async throws { guard let vectorData = "{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1564,7 +1574,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector153WireConformanceToRequestToolsBindingsStripped() async throws { + func testVector154WireConformanceToRequestToolsBindingsStripped() async throws { guard let vectorData = "{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1574,7 +1584,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector154WireConformanceToRequestToolsFunctionWire() async throws { + func testVector155WireConformanceToRequestToolsFunctionWire() async throws { guard let vectorData = "{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1584,7 +1594,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector155WireConformanceToRequestToolsNullWhenEmpty() async throws { + func testVector156WireConformanceToRequestToolsNullWhenEmpty() async throws { guard let vectorData = "{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1594,7 +1604,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector156WireConformanceToRequestToolsStrictMode() async throws { + func testVector157WireConformanceToRequestToolsStrictMode() async throws { guard let vectorData = "{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { diff --git a/runtime/typescript/packages/core/src/core/endpoint-trust.ts b/runtime/typescript/packages/core/src/core/endpoint-trust.ts index 0bf2f0efc..967119671 100644 --- a/runtime/typescript/packages/core/src/core/endpoint-trust.ts +++ b/runtime/typescript/packages/core/src/core/endpoint-trust.ts @@ -27,7 +27,8 @@ export const BUILTIN_TRUSTED_ENDPOINTS: readonly string[] = Object.freeze([ const DEFAULT_PORTS: Record = { http: 80, https: 443 }; const CREDENTIAL_KEYS: Record = { key: "apiKey", - oauth: "clientSecret", + + }; const DESTINATION_KEYS = ["endpoint", "tokenUrl"] as const; diff --git a/runtime/typescript/packages/core/src/model/vector-conformance.ts b/runtime/typescript/packages/core/src/model/vector-conformance.ts index a84e4188c..bf46731ad 100644 --- a/runtime/typescript/packages/core/src/model/vector-conformance.ts +++ b/runtime/typescript/packages/core/src/model/vector-conformance.ts @@ -924,6 +924,21 @@ export async function runLoadConformanceConformance( "untrusted_endpoint_no_key_rejected: expected an error", ); } + // vector: untrusted_endpoint_oauth_literal_secret_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_oauth_literal_secret_rejected: expected an error", + ); + } // vector: untrusted_endpoint_percent_encoded_host_rejected { const sourceJson = "null"; diff --git a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts index 2797b945a..2af293a40 100644 --- a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts +++ b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts @@ -435,6 +435,12 @@ describe("callable vector conformance", () => { ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:untrusted_endpoint_oauth_literal_secret_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_oauth_literal_secret_rejected","description":"kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials","stage":"load","input":{"frontmatter":{"name":"untrusted-oauth-literal","model":{"id":"gpt-4","provider":"anthropic","connection":{"kind":"oauth","endpoint":"https://attacker.example.com","clientId":"decoy","clientSecret":"decoy","tokenUrl":"https://login.microsoftonline.com/t/oauth2/v2.0/token"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("LoadConformance.load:untrusted_endpoint_percent_encoded_host_rejected", async () => { const vector = JSON.parse( '{"name":"untrusted_endpoint_percent_encoded_host_rejected","description":"Hosts are compared without percent-decoding, so an encoded host never matches","stage":"load","input":{"frontmatter":{"name":"untrusted-percent-encoded-host","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com%2eevil.example.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', diff --git a/schema/model/conformance/vectors/load.tsp b/schema/model/conformance/vectors/load.tsp index 396b5ec11..1ce087088 100644 --- a/schema/model/conformance/vectors/load.tsp +++ b/schema/model/conformance/vectors/load.tsp @@ -377,6 +377,30 @@ const LoadVectors = """ "kind": "untrusted_endpoint" } }, + { + "name": "untrusted_endpoint_oauth_literal_secret_rejected", + "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-oauth-literal", + "model": { + "id": "gpt-4", + "provider": "anthropic", + "connection": { + "kind": "oauth", + "endpoint": "https://attacker.example.com", + "clientId": "decoy", + "clientSecret": "decoy", + "tokenUrl": "https://login.microsoftonline.com/t/oauth2/v2.0/token" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, { "name": "untrusted_endpoint_backslash_rejected", "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", diff --git a/schema/tsp-output/.typra-generated/vectors.json b/schema/tsp-output/.typra-generated/vectors.json index 15ac46f74..1ced3bc64 100644 --- a/schema/tsp-output/.typra-generated/vectors.json +++ b/schema/tsp-output/.typra-generated/vectors.json @@ -2494,6 +2494,42 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_oauth_literal_secret_rejected", + "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-oauth-literal", + "model": { + "id": "gpt-4", + "provider": "anthropic", + "connection": { + "kind": "oauth", + "endpoint": "https://attacker.example.com", + "clientId": "decoy", + "clientSecret": "decoy", + "tokenUrl": "https://login.microsoftonline.com/t/oauth2/v2.0/token" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, { "contract": "LoadConformance", "namespace": "Prompty.Conformance.Seams", diff --git a/spec/spec.md b/spec/spec.md index 98e3ecedb..927e9d6dd 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -295,11 +295,11 @@ A file MUST NOT be able to direct ambient credentials to a destination it chose. Loaders MUST enforce this at load time (§4.2 step 5) on `model.connection`: 1. Skip the check when `connection` is absent or `kind` is `reference`. -2. The **credential is file-supplied** only when the field the declared `kind` consumes - is a non-empty literal string (not a `${...}` reference): `apiKey` for `kind: key`, - `clientSecret` for `kind: oauth`. Every other case is ambient, including `${env:}` or - `${file:}` credentials, any other `kind` (e.g. `foundry`, `anonymous`, custom), and - credential fields the declared `kind` does not consume (e.g. `apiKey` on `kind: foundry`). +2. The **credential is file-supplied** only when `kind` is `key` and `apiKey` is a + non-empty literal string (not a `${...}` reference). Every other case is ambient, + including `${env:}` or `${file:}` keys, any other `kind` (e.g. `foundry`, `oauth`, + `anonymous`, custom), and an `apiKey` on any other `kind`. (`oauth` is ambient because + executors do not consume `clientSecret` and fall back to ambient credentials.) 3. Each **destination** (`endpoint`, and `tokenUrl` when present) is **host-controlled** when it is absent, or is `${env:VAR}` / `${env:VAR:default}` and `VAR` is set. A literal value, a `${file:}` value (always untrusted), or an `${env:}` default used diff --git a/web/src/content/docs/specification/file-format.mdx b/web/src/content/docs/specification/file-format.mdx index 60ffc5464..076793169 100644 --- a/web/src/content/docs/specification/file-format.mdx +++ b/web/src/content/docs/specification/file-format.mdx @@ -136,8 +136,7 @@ Connection types are discriminated by the `kind` field: A `.prompty` file is untrusted input, so it MUST NOT be able to direct **ambient credentials** (`DefaultAzureCredential` tokens, env API keys) to a destination it chose. -Unless the connection supplies its own literal credential (`apiKey` on `kind: key`, -`clientSecret` on `kind: oauth`), a file-controlled `endpoint` or `tokenUrl` (a literal, a +Unless the connection supplies its own literal credential (`apiKey` on `kind: key`), a file-controlled `endpoint` or `tokenUrl` (a literal, a `${file:}` value, or a used `${env:VAR:default}` fallback) must match a trusted origin, or loading fails with `untrusted_endpoint`. `kind: reference` and `${env:VAR}` endpoints are host-controlled and pass. From 0a12bf048b0662b4418dd27c3992d048c89b20be Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 18:16:51 -0700 Subject: [PATCH 14/15] style(typescript): format endpoint trust module Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/typescript/packages/core/src/core/endpoint-trust.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/runtime/typescript/packages/core/src/core/endpoint-trust.ts b/runtime/typescript/packages/core/src/core/endpoint-trust.ts index 967119671..1e2d1df24 100644 --- a/runtime/typescript/packages/core/src/core/endpoint-trust.ts +++ b/runtime/typescript/packages/core/src/core/endpoint-trust.ts @@ -27,8 +27,6 @@ export const BUILTIN_TRUSTED_ENDPOINTS: readonly string[] = Object.freeze([ const DEFAULT_PORTS: Record = { http: 80, https: 443 }; const CREDENTIAL_KEYS: Record = { key: "apiKey", - - }; const DESTINATION_KEYS = ["endpoint", "tokenUrl"] as const; From 2c18c2dafe3ff77ea57cb3e6ac60ad5806a97e6c Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 20 Oct 2026 19:10:21 -0700 Subject: [PATCH 15/15] fix(spec): treat blank literal api keys as ambient in endpoint trust Some executors trim the connection apiKey and fall back to environment credentials when it is empty. A whitespace-only literal key therefore must not count as file-supplied. Update spec 2.5.1, add a conformance vector, and align the gate in all runtimes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Model/VectorConformanceTests.cs | 232 +++++----- runtime/csharp/Prompty.Core/EndpointTrust.cs | 5 +- .../Prompty.Core/Model/VectorConformance.cs | 14 + runtime/go/prompty/model/endpoint_trust.go | 3 +- .../prompty/model/vector_conformance_test.go | 231 +++++----- .../vectorconformance/vector_conformance.go | 11 + .../com/microsoft/prompty/EndpointTrust.java | 3 +- .../prompty/model/VectorConformance.java | 14 + .../prompty/model/VectorConformanceTests.java | 408 +++++++++--------- .../prompty/prompty/core/endpoint_trust.py | 3 +- .../prompty/model/vector_conformance.py | 10 + .../tests/model/test_vector_conformance.py | 230 +++++----- .../rust/prompty/src/loader/endpoint_trust.rs | 3 +- .../prompty/src/model/vector_conformance.rs | 14 + .../tests/model/vector_conformance_test.rs | 207 ++++----- .../Sources/PromptyModel/EndpointTrust.swift | 3 +- .../PromptyModel/vector_conformance.swift | 17 + .../VectorConformanceTests.swift | 210 ++++----- .../packages/core/src/core/endpoint-trust.ts | 5 +- .../core/src/model/vector-conformance.ts | 15 + .../tests/model/vector-conformance.test.ts | 6 + schema/model/conformance/vectors/load.tsp | 22 + .../tsp-output/.typra-generated/vectors.json | 34 ++ spec/spec.md | 4 +- 24 files changed, 994 insertions(+), 710 deletions(-) diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs index aea8e4ab9..d3aba3b54 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs @@ -2148,7 +2148,39 @@ public async Task Vector57LoadConformanceLoadUntrustedEndpointBackslashRejected( } [Fact] - public async Task Vector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() + public async Task Vector58LoadConformanceLoadUntrustedEndpointBlankLiteralKeyRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_blank_literal_key_rejected", + "description": "a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-blank-key", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": " " + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector59LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() { string vectorJson = """ { @@ -2179,7 +2211,7 @@ public async Task Vector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected } [Fact] - public async Task Vector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected() + public async Task Vector60LoadConformanceLoadUntrustedEndpointEnvKeyRejected() { string vectorJson = """ { @@ -2213,7 +2245,7 @@ public async Task Vector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected() } [Fact] - public async Task Vector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected() + public async Task Vector61LoadConformanceLoadUntrustedEndpointFileConnectionRejected() { string vectorJson = """ { @@ -2246,7 +2278,7 @@ public async Task Vector60LoadConformanceLoadUntrustedEndpointFileConnectionReje } [Fact] - public async Task Vector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() + public async Task Vector62LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() { string vectorJson = """ { @@ -2276,7 +2308,7 @@ public async Task Vector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRe } [Fact] - public async Task Vector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() + public async Task Vector63LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() { string vectorJson = """ { @@ -2306,7 +2338,7 @@ public async Task Vector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejecte } [Fact] - public async Task Vector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() + public async Task Vector64LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() { string vectorJson = """ { @@ -2337,7 +2369,7 @@ public async Task Vector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecre } [Fact] - public async Task Vector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() + public async Task Vector65LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() { string vectorJson = """ { @@ -2368,7 +2400,7 @@ public async Task Vector64LoadConformanceLoadUntrustedEndpointMismatchedCredenti } [Fact] - public async Task Vector65LoadConformanceLoadUntrustedEndpointNoKeyRejected() + public async Task Vector66LoadConformanceLoadUntrustedEndpointNoKeyRejected() { string vectorJson = """ { @@ -2398,7 +2430,7 @@ public async Task Vector65LoadConformanceLoadUntrustedEndpointNoKeyRejected() } [Fact] - public async Task Vector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() + public async Task Vector67LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() { string vectorJson = """ { @@ -2432,7 +2464,7 @@ public async Task Vector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecret } [Fact] - public async Task Vector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() + public async Task Vector68LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() { string vectorJson = """ { @@ -2462,7 +2494,7 @@ public async Task Vector67LoadConformanceLoadUntrustedEndpointPercentEncodedHost } [Fact] - public async Task Vector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected() + public async Task Vector69LoadConformanceLoadUntrustedEndpointPortMismatchRejected() { string vectorJson = """ { @@ -2492,7 +2524,7 @@ public async Task Vector68LoadConformanceLoadUntrustedEndpointPortMismatchReject } [Fact] - public async Task Vector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() + public async Task Vector70LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() { string vectorJson = """ { @@ -2522,7 +2554,7 @@ public async Task Vector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejec } [Fact] - public async Task Vector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() + public async Task Vector71LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() { string vectorJson = """ { @@ -2552,7 +2584,7 @@ public async Task Vector70LoadConformanceLoadUntrustedEndpointSchemeMismatchReje } [Fact] - public async Task Vector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() + public async Task Vector72LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() { string vectorJson = """ { @@ -2582,7 +2614,7 @@ public async Task Vector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRej } [Fact] - public async Task Vector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() + public async Task Vector73LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() { string vectorJson = """ { @@ -2615,7 +2647,7 @@ public async Task Vector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejecte } [Fact] - public async Task Vector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected() + public async Task Vector74LoadConformanceLoadUntrustedEndpointTrailingDotRejected() { string vectorJson = """ { @@ -2645,7 +2677,7 @@ public async Task Vector73LoadConformanceLoadUntrustedEndpointTrailingDotRejecte } [Fact] - public async Task Vector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() + public async Task Vector75LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() { string vectorJson = """ { @@ -2675,7 +2707,7 @@ public async Task Vector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRe } [Fact] - public async Task Vector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() + public async Task Vector76LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() { string vectorJson = """ { @@ -2705,7 +2737,7 @@ public async Task Vector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeR } [Fact] - public async Task Vector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected() + public async Task Vector77LoadConformanceLoadUntrustedEndpointWildcardApexRejected() { string vectorJson = """ { @@ -2735,7 +2767,7 @@ public async Task Vector76LoadConformanceLoadUntrustedEndpointWildcardApexReject } [Fact] - public async Task Vector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() + public async Task Vector78LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() { string vectorJson = """ { @@ -2765,7 +2797,7 @@ public async Task Vector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRe } [Fact] - public async Task Vector78MemoryConformanceOperateClearByCategoryAndAll() + public async Task Vector79MemoryConformanceOperateClearByCategoryAndAll() { string vectorJson = """ { @@ -2819,7 +2851,7 @@ public async Task Vector78MemoryConformanceOperateClearByCategoryAndAll() } [Fact] - public async Task Vector79MemoryConformanceOperateFormatCoreAndRecallResults() + public async Task Vector80MemoryConformanceOperateFormatCoreAndRecallResults() { string vectorJson = """ { @@ -2861,7 +2893,7 @@ public async Task Vector79MemoryConformanceOperateFormatCoreAndRecallResults() } [Fact] - public async Task Vector80MemoryConformanceOperateRecallEmptyQueryAndLimit() + public async Task Vector81MemoryConformanceOperateRecallEmptyQueryAndLimit() { string vectorJson = """ { @@ -2916,7 +2948,7 @@ public async Task Vector80MemoryConformanceOperateRecallEmptyQueryAndLimit() } [Fact] - public async Task Vector81MemoryConformanceOperateRecallRanksByWeightedScore() + public async Task Vector82MemoryConformanceOperateRecallRanksByWeightedScore() { string vectorJson = """ { @@ -2975,7 +3007,7 @@ public async Task Vector81MemoryConformanceOperateRecallRanksByWeightedScore() } [Fact] - public async Task Vector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() + public async Task Vector83MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() { string vectorJson = """ { @@ -3033,7 +3065,7 @@ public async Task Vector82MemoryConformanceOperateRecallScoreTiesPreserveInserti } [Fact] - public async Task Vector83MemoryConformanceOperateRecallTagsAndCoreBoost() + public async Task Vector84MemoryConformanceOperateRecallTagsAndCoreBoost() { string vectorJson = """ { @@ -3094,7 +3126,7 @@ public async Task Vector83MemoryConformanceOperateRecallTagsAndCoreBoost() } [Fact] - public async Task Vector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() + public async Task Vector85MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() { string vectorJson = """ { @@ -3155,7 +3187,7 @@ public async Task Vector84MemoryConformanceOperateRecallTokenizationPunctuationU } [Fact] - public async Task Vector85MemoryConformanceOperateRememberDedupsCoreByTags() + public async Task Vector86MemoryConformanceOperateRememberDedupsCoreByTags() { string vectorJson = """ { @@ -3219,7 +3251,7 @@ public async Task Vector85MemoryConformanceOperateRememberDedupsCoreByTags() } [Fact] - public async Task Vector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() + public async Task Vector87MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() { string vectorJson = """ { @@ -3275,7 +3307,7 @@ public async Task Vector86MemoryConformanceOperateRememberDedupsCoreMissingAndEm } [Fact] - public async Task Vector87MemoryConformanceOperateRememberEvictsArchivalFirst() + public async Task Vector88MemoryConformanceOperateRememberEvictsArchivalFirst() { string vectorJson = """ { @@ -3336,7 +3368,7 @@ public async Task Vector87MemoryConformanceOperateRememberEvictsArchivalFirst() } [Fact] - public async Task Vector88MemoryConformanceOperateSnapshotPortRoundTrip() + public async Task Vector89MemoryConformanceOperateSnapshotPortRoundTrip() { string vectorJson = """ { @@ -3376,7 +3408,7 @@ public async Task Vector88MemoryConformanceOperateSnapshotPortRoundTrip() } [Fact] - public async Task Vector89MemoryConformanceOperateUpdateIndexOutOfRangeError() + public async Task Vector90MemoryConformanceOperateUpdateIndexOutOfRangeError() { string vectorJson = """ { @@ -3410,7 +3442,7 @@ public async Task Vector89MemoryConformanceOperateUpdateIndexOutOfRangeError() } [Fact] - public async Task Vector90TurnConformanceReplayMaxIterations() + public async Task Vector91TurnConformanceReplayMaxIterations() { string vectorJson = """ { @@ -3447,7 +3479,7 @@ public async Task Vector90TurnConformanceReplayMaxIterations() } [Fact] - public async Task Vector91TurnConformanceReplayNoTool() + public async Task Vector92TurnConformanceReplayNoTool() { string vectorJson = """ { @@ -3480,7 +3512,7 @@ public async Task Vector91TurnConformanceReplayNoTool() } [Fact] - public async Task Vector92TurnConformanceReplayPermissionDenied() + public async Task Vector93TurnConformanceReplayPermissionDenied() { string vectorJson = """ { @@ -3516,7 +3548,7 @@ public async Task Vector92TurnConformanceReplayPermissionDenied() } [Fact] - public async Task Vector93TurnConformanceReplayToolFailure() + public async Task Vector94TurnConformanceReplayToolFailure() { string vectorJson = """ { @@ -3554,7 +3586,7 @@ public async Task Vector93TurnConformanceReplayToolFailure() } [Fact] - public async Task Vector94TurnConformanceReplayToolSuccess() + public async Task Vector95TurnConformanceReplayToolSuccess() { string vectorJson = """ { @@ -3592,7 +3624,7 @@ public async Task Vector94TurnConformanceReplayToolSuccess() } [Fact] - public async Task Vector95TurnConformanceRunAssistantToolCallsMetadata() + public async Task Vector96TurnConformanceRunAssistantToolCallsMetadata() { string vectorJson = """ { @@ -3746,7 +3778,7 @@ public async Task Vector95TurnConformanceRunAssistantToolCallsMetadata() } [Fact] - public async Task Vector96TurnConformanceRunAsyncToolFunction() + public async Task Vector97TurnConformanceRunAsyncToolFunction() { string vectorJson = """ { @@ -3858,7 +3890,7 @@ public async Task Vector96TurnConformanceRunAsyncToolFunction() } [Fact] - public async Task Vector97TurnConformanceRunBindingsInjected() + public async Task Vector98TurnConformanceRunBindingsInjected() { string vectorJson = """ { @@ -3990,7 +4022,7 @@ public async Task Vector97TurnConformanceRunBindingsInjected() } [Fact] - public async Task Vector98TurnConformanceRunCancellationBeforeLlm() + public async Task Vector99TurnConformanceRunCancellationBeforeLlm() { string vectorJson = """ { @@ -4053,7 +4085,7 @@ public async Task Vector98TurnConformanceRunCancellationBeforeLlm() } [Fact] - public async Task Vector99TurnConformanceRunCancellationBetweenIterations() + public async Task Vector100TurnConformanceRunCancellationBetweenIterations() { string vectorJson = """ { @@ -4185,7 +4217,7 @@ public async Task Vector99TurnConformanceRunCancellationBetweenIterations() } [Fact] - public async Task Vector100TurnConformanceRunCancellationBetweenTools() + public async Task Vector101TurnConformanceRunCancellationBetweenTools() { string vectorJson = """ { @@ -4326,7 +4358,7 @@ public async Task Vector100TurnConformanceRunCancellationBetweenTools() } [Fact] - public async Task Vector101TurnConformanceRunContextNoTrimWhenFits() + public async Task Vector102TurnConformanceRunContextNoTrimWhenFits() { string vectorJson = """ { @@ -4399,7 +4431,7 @@ public async Task Vector101TurnConformanceRunContextNoTrimWhenFits() } [Fact] - public async Task Vector102TurnConformanceRunContextPreservesSystemMessages() + public async Task Vector103TurnConformanceRunContextPreservesSystemMessages() { string vectorJson = """ { @@ -4508,7 +4540,7 @@ public async Task Vector102TurnConformanceRunContextPreservesSystemMessages() } [Fact] - public async Task Vector103TurnConformanceRunContextTrimBasic() + public async Task Vector104TurnConformanceRunContextTrimBasic() { string vectorJson = """ { @@ -4668,7 +4700,7 @@ public async Task Vector103TurnConformanceRunContextTrimBasic() } [Fact] - public async Task Vector104TurnConformanceRunEmptyToolResult() + public async Task Vector105TurnConformanceRunEmptyToolResult() { string vectorJson = """ { @@ -4785,7 +4817,7 @@ public async Task Vector104TurnConformanceRunEmptyToolResult() } [Fact] - public async Task Vector105TurnConformanceRunEventsBasicToolLoop() + public async Task Vector106TurnConformanceRunEventsBasicToolLoop() { string vectorJson = """ { @@ -4933,7 +4965,7 @@ public async Task Vector105TurnConformanceRunEventsBasicToolLoop() } [Fact] - public async Task Vector106TurnConformanceRunEventsErrorLogged() + public async Task Vector107TurnConformanceRunEventsErrorLogged() { string vectorJson = """ { @@ -5061,7 +5093,7 @@ public async Task Vector106TurnConformanceRunEventsErrorLogged() } [Fact] - public async Task Vector107TurnConformanceRunEventsNoTools() + public async Task Vector108TurnConformanceRunEventsNoTools() { string vectorJson = """ { @@ -5141,7 +5173,7 @@ public async Task Vector107TurnConformanceRunEventsNoTools() } [Fact] - public async Task Vector108TurnConformanceRunGuardrailAllPass() + public async Task Vector109TurnConformanceRunGuardrailAllPass() { string vectorJson = """ { @@ -5266,7 +5298,7 @@ public async Task Vector108TurnConformanceRunGuardrailAllPass() } [Fact] - public async Task Vector109TurnConformanceRunGuardrailInputDeny() + public async Task Vector110TurnConformanceRunGuardrailInputDeny() { string vectorJson = """ { @@ -5324,7 +5356,7 @@ public async Task Vector109TurnConformanceRunGuardrailInputDeny() } [Fact] - public async Task Vector110TurnConformanceRunGuardrailOutputDeny() + public async Task Vector111TurnConformanceRunGuardrailOutputDeny() { string vectorJson = """ { @@ -5404,7 +5436,7 @@ public async Task Vector110TurnConformanceRunGuardrailOutputDeny() } [Fact] - public async Task Vector111TurnConformanceRunGuardrailToolDeny() + public async Task Vector112TurnConformanceRunGuardrailToolDeny() { string vectorJson = """ { @@ -5560,7 +5592,7 @@ public async Task Vector111TurnConformanceRunGuardrailToolDeny() } [Fact] - public async Task Vector112TurnConformanceRunMaxIterationsExceeded() + public async Task Vector113TurnConformanceRunMaxIterationsExceeded() { string vectorJson = """ { @@ -6072,7 +6104,7 @@ public async Task Vector112TurnConformanceRunMaxIterationsExceeded() } [Fact] - public async Task Vector113TurnConformanceRunMultiTurnToolCalls() + public async Task Vector114TurnConformanceRunMultiTurnToolCalls() { string vectorJson = """ { @@ -6251,7 +6283,7 @@ public async Task Vector113TurnConformanceRunMultiTurnToolCalls() } [Fact] - public async Task Vector114TurnConformanceRunMultipleToolCallsSingleTurn() + public async Task Vector115TurnConformanceRunMultipleToolCallsSingleTurn() { string vectorJson = """ { @@ -6434,7 +6466,7 @@ public async Task Vector114TurnConformanceRunMultipleToolCallsSingleTurn() } [Fact] - public async Task Vector115TurnConformanceRunNoToolCalls() + public async Task Vector116TurnConformanceRunNoToolCalls() { string vectorJson = """ { @@ -6504,7 +6536,7 @@ public async Task Vector115TurnConformanceRunNoToolCalls() } [Fact] - public async Task Vector116TurnConformanceRunParallelToolsBasic() + public async Task Vector117TurnConformanceRunParallelToolsBasic() { string vectorJson = """ { @@ -6679,7 +6711,7 @@ public async Task Vector116TurnConformanceRunParallelToolsBasic() } [Fact] - public async Task Vector117TurnConformanceRunParallelToolsWithGuardrailDeny() + public async Task Vector118TurnConformanceRunParallelToolsWithGuardrailDeny() { string vectorJson = """ { @@ -6872,7 +6904,7 @@ public async Task Vector117TurnConformanceRunParallelToolsWithGuardrailDeny() } [Fact] - public async Task Vector118TurnConformanceRunParallelToolsWithToolError() + public async Task Vector119TurnConformanceRunParallelToolsWithToolError() { string vectorJson = """ { @@ -7047,7 +7079,7 @@ public async Task Vector118TurnConformanceRunParallelToolsWithToolError() } [Fact] - public async Task Vector119TurnConformanceRunSingleToolCall() + public async Task Vector120TurnConformanceRunSingleToolCall() { string vectorJson = """ { @@ -7196,7 +7228,7 @@ public async Task Vector119TurnConformanceRunSingleToolCall() } [Fact] - public async Task Vector120TurnConformanceRunSteeringInjectMessage() + public async Task Vector121TurnConformanceRunSteeringInjectMessage() { string vectorJson = """ { @@ -7406,7 +7438,7 @@ public async Task Vector120TurnConformanceRunSteeringInjectMessage() } [Fact] - public async Task Vector121TurnConformanceRunSteeringMultipleMessages() + public async Task Vector122TurnConformanceRunSteeringMultipleMessages() { string vectorJson = """ { @@ -7578,7 +7610,7 @@ public async Task Vector121TurnConformanceRunSteeringMultipleMessages() } [Fact] - public async Task Vector122TurnConformanceRunToolNotRegisteredError() + public async Task Vector123TurnConformanceRunToolNotRegisteredError() { string vectorJson = """ { @@ -7664,7 +7696,7 @@ public async Task Vector122TurnConformanceRunToolNotRegisteredError() } [Fact] - public async Task Vector123TurnConformanceRunToolResultMessageFormat() + public async Task Vector124TurnConformanceRunToolResultMessageFormat() { string vectorJson = """ { @@ -7787,7 +7819,7 @@ public async Task Vector123TurnConformanceRunToolResultMessageFormat() } [Fact] - public async Task Vector124TurnConformanceRunTurnCancelBeforeContext() + public async Task Vector125TurnConformanceRunTurnCancelBeforeContext() { string vectorJson = """ { @@ -7821,7 +7853,7 @@ public async Task Vector124TurnConformanceRunTurnCancelBeforeContext() } [Fact] - public async Task Vector125TurnConformanceRunTurnDelegatedProviderState() + public async Task Vector126TurnConformanceRunTurnDelegatedProviderState() { string vectorJson = """ { @@ -7890,7 +7922,7 @@ public async Task Vector125TurnConformanceRunTurnDelegatedProviderState() } [Fact] - public async Task Vector126TurnConformanceRunTurnFinalOutput() + public async Task Vector127TurnConformanceRunTurnFinalOutput() { string vectorJson = """ { @@ -7937,7 +7969,7 @@ public async Task Vector126TurnConformanceRunTurnFinalOutput() } [Fact] - public async Task Vector127TurnConformanceRunTurnMemoryContextIsModelVisible() + public async Task Vector128TurnConformanceRunTurnMemoryContextIsModelVisible() { string vectorJson = """ { @@ -8001,7 +8033,7 @@ public async Task Vector127TurnConformanceRunTurnMemoryContextIsModelVisible() } [Fact] - public async Task Vector128TurnConformanceRunTurnOrderedToolRound() + public async Task Vector129TurnConformanceRunTurnOrderedToolRound() { string vectorJson = """ { @@ -8094,7 +8126,7 @@ public async Task Vector128TurnConformanceRunTurnOrderedToolRound() } [Fact] - public async Task Vector129TurnConformanceRunTurnPermissionDenialIsModelVisible() + public async Task Vector130TurnConformanceRunTurnPermissionDenialIsModelVisible() { string vectorJson = """ { @@ -8147,7 +8179,7 @@ public async Task Vector129TurnConformanceRunTurnPermissionDenialIsModelVisible( } [Fact] - public async Task Vector130WireConformanceToRequestAnthropicImageFormat() + public async Task Vector131WireConformanceToRequestAnthropicImageFormat() { string vectorJson = """ { @@ -8219,7 +8251,7 @@ public async Task Vector130WireConformanceToRequestAnthropicImageFormat() } [Fact] - public async Task Vector131WireConformanceToRequestAnthropicMaxTokensRequired() + public async Task Vector132WireConformanceToRequestAnthropicMaxTokensRequired() { string vectorJson = """ { @@ -8276,7 +8308,7 @@ public async Task Vector131WireConformanceToRequestAnthropicMaxTokensRequired() } [Fact] - public async Task Vector132WireConformanceToRequestAnthropicOptions() + public async Task Vector133WireConformanceToRequestAnthropicOptions() { string vectorJson = """ { @@ -8347,7 +8379,7 @@ public async Task Vector132WireConformanceToRequestAnthropicOptions() } [Fact] - public async Task Vector133WireConformanceToRequestAnthropicSystemSeparate() + public async Task Vector134WireConformanceToRequestAnthropicSystemSeparate() { string vectorJson = """ { @@ -8416,7 +8448,7 @@ public async Task Vector133WireConformanceToRequestAnthropicSystemSeparate() } [Fact] - public async Task Vector134WireConformanceToRequestAnthropicToolWire() + public async Task Vector135WireConformanceToRequestAnthropicToolWire() { string vectorJson = """ { @@ -8503,7 +8535,7 @@ public async Task Vector134WireConformanceToRequestAnthropicToolWire() } [Fact] - public async Task Vector135WireConformanceToRequestChatAudioMp3() + public async Task Vector136WireConformanceToRequestChatAudioMp3() { string vectorJson = """ { @@ -8563,7 +8595,7 @@ public async Task Vector135WireConformanceToRequestChatAudioMp3() } [Fact] - public async Task Vector136WireConformanceToRequestChatAudioPart() + public async Task Vector137WireConformanceToRequestChatAudioPart() { string vectorJson = """ { @@ -8623,7 +8655,7 @@ public async Task Vector136WireConformanceToRequestChatAudioPart() } [Fact] - public async Task Vector137WireConformanceToRequestChatImageBase64() + public async Task Vector138WireConformanceToRequestChatImageBase64() { string vectorJson = """ { @@ -8689,7 +8721,7 @@ public async Task Vector137WireConformanceToRequestChatImageBase64() } [Fact] - public async Task Vector138WireConformanceToRequestChatImagePart() + public async Task Vector139WireConformanceToRequestChatImagePart() { string vectorJson = """ { @@ -8755,7 +8787,7 @@ public async Task Vector138WireConformanceToRequestChatImagePart() } [Fact] - public async Task Vector139WireConformanceToRequestChatMultipartContent() + public async Task Vector140WireConformanceToRequestChatMultipartContent() { string vectorJson = """ { @@ -8821,7 +8853,7 @@ public async Task Vector139WireConformanceToRequestChatMultipartContent() } [Fact] - public async Task Vector140WireConformanceToRequestChatSimple() + public async Task Vector141WireConformanceToRequestChatSimple() { string vectorJson = """ { @@ -8885,7 +8917,7 @@ public async Task Vector140WireConformanceToRequestChatSimple() } [Fact] - public async Task Vector141WireConformanceToRequestChatSingleTextOptimized() + public async Task Vector142WireConformanceToRequestChatSingleTextOptimized() { string vectorJson = """ { @@ -8936,7 +8968,7 @@ public async Task Vector141WireConformanceToRequestChatSingleTextOptimized() } [Fact] - public async Task Vector142WireConformanceToRequestChatWithOptions() + public async Task Vector143WireConformanceToRequestChatWithOptions() { string vectorJson = """ { @@ -9006,7 +9038,7 @@ public async Task Vector142WireConformanceToRequestChatWithOptions() } [Fact] - public async Task Vector143WireConformanceToRequestEmbeddingWire() + public async Task Vector144WireConformanceToRequestEmbeddingWire() { string vectorJson = """ { @@ -9052,7 +9084,7 @@ public async Task Vector143WireConformanceToRequestEmbeddingWire() } [Fact] - public async Task Vector144WireConformanceToRequestImageWire() + public async Task Vector145WireConformanceToRequestImageWire() { string vectorJson = """ { @@ -9098,7 +9130,7 @@ public async Task Vector144WireConformanceToRequestImageWire() } [Fact] - public async Task Vector145WireConformanceToRequestKindToJsonTypeMapping() + public async Task Vector146WireConformanceToRequestKindToJsonTypeMapping() { string vectorJson = """ { @@ -9219,7 +9251,7 @@ public async Task Vector145WireConformanceToRequestKindToJsonTypeMapping() } [Fact] - public async Task Vector146WireConformanceToRequestOptionsAdditionalProperties() + public async Task Vector147WireConformanceToRequestOptionsAdditionalProperties() { string vectorJson = """ { @@ -9275,7 +9307,7 @@ public async Task Vector146WireConformanceToRequestOptionsAdditionalProperties() } [Fact] - public async Task Vector147WireConformanceToRequestOptionsMaxCompletionTokens() + public async Task Vector148WireConformanceToRequestOptionsMaxCompletionTokens() { string vectorJson = """ { @@ -9329,7 +9361,7 @@ public async Task Vector147WireConformanceToRequestOptionsMaxCompletionTokens() } [Fact] - public async Task Vector148WireConformanceToRequestOptionsStopSequences() + public async Task Vector149WireConformanceToRequestOptionsStopSequences() { string vectorJson = """ { @@ -9389,7 +9421,7 @@ public async Task Vector148WireConformanceToRequestOptionsStopSequences() } [Fact] - public async Task Vector149WireConformanceToRequestResponsesSimple() + public async Task Vector150WireConformanceToRequestResponsesSimple() { string vectorJson = """ { @@ -9450,7 +9482,7 @@ public async Task Vector149WireConformanceToRequestResponsesSimple() } [Fact] - public async Task Vector150WireConformanceToRequestResponsesStructuredOutput() + public async Task Vector151WireConformanceToRequestResponsesStructuredOutput() { string vectorJson = """ { @@ -9539,7 +9571,7 @@ public async Task Vector150WireConformanceToRequestResponsesStructuredOutput() } [Fact] - public async Task Vector151WireConformanceToRequestResponsesWithTools() + public async Task Vector152WireConformanceToRequestResponsesWithTools() { string vectorJson = """ { @@ -9621,7 +9653,7 @@ public async Task Vector151WireConformanceToRequestResponsesWithTools() } [Fact] - public async Task Vector152WireConformanceToRequestStructuredOutput() + public async Task Vector153WireConformanceToRequestStructuredOutput() { string vectorJson = """ { @@ -9706,7 +9738,7 @@ public async Task Vector152WireConformanceToRequestStructuredOutput() } [Fact] - public async Task Vector153WireConformanceToRequestStructuredOutputNestedOptional() + public async Task Vector154WireConformanceToRequestStructuredOutputNestedOptional() { string vectorJson = """ { @@ -9810,7 +9842,7 @@ public async Task Vector153WireConformanceToRequestStructuredOutputNestedOptiona } [Fact] - public async Task Vector154WireConformanceToRequestToolsBindingsStripped() + public async Task Vector155WireConformanceToRequestToolsBindingsStripped() { string vectorJson = """ { @@ -9904,7 +9936,7 @@ public async Task Vector154WireConformanceToRequestToolsBindingsStripped() } [Fact] - public async Task Vector155WireConformanceToRequestToolsFunctionWire() + public async Task Vector156WireConformanceToRequestToolsFunctionWire() { string vectorJson = """ { @@ -9988,7 +10020,7 @@ public async Task Vector155WireConformanceToRequestToolsFunctionWire() } [Fact] - public async Task Vector156WireConformanceToRequestToolsNullWhenEmpty() + public async Task Vector157WireConformanceToRequestToolsNullWhenEmpty() { string vectorJson = """ { @@ -10039,7 +10071,7 @@ public async Task Vector156WireConformanceToRequestToolsNullWhenEmpty() } [Fact] - public async Task Vector157WireConformanceToRequestToolsStrictMode() + public async Task Vector158WireConformanceToRequestToolsStrictMode() { string vectorJson = """ { diff --git a/runtime/csharp/Prompty.Core/EndpointTrust.cs b/runtime/csharp/Prompty.Core/EndpointTrust.cs index e3a07f29e..afa501c4b 100644 --- a/runtime/csharp/Prompty.Core/EndpointTrust.cs +++ b/runtime/csharp/Prompty.Core/EndpointTrust.cs @@ -119,13 +119,12 @@ private static bool FileSuppliedCredential(IDictionary connection) var key = Get(connection, "kind") switch { "key" => "apiKey", - - _ => null, }; if (key is null) return false; - return Get(connection, key) is string value && value.Length > 0 && !IsReference(value); + // Blank keys count as absent: executors that trim fall back to ambient credentials. + return Get(connection, key) is string value && !string.IsNullOrWhiteSpace(value) && !IsReference(value); } /// Return the effective destination if the file chose it, else null. diff --git a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs index b24b0fd4b..381e6cf18 100644 --- a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs +++ b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs @@ -471,6 +471,20 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea } if (caught is null) throw new SeamConformanceException("untrusted_endpoint_backslash_rejected: expected an error"); } + { + // vector: untrusted_endpoint_blank_literal_key_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_blank_literal_key_rejected: expected an error"); + } { // vector: untrusted_endpoint_env_default_rejected var source = JsonSerializer.Deserialize("null")!; diff --git a/runtime/go/prompty/model/endpoint_trust.go b/runtime/go/prompty/model/endpoint_trust.go index 5460d52c7..e53736902 100644 --- a/runtime/go/prompty/model/endpoint_trust.go +++ b/runtime/go/prompty/model/endpoint_trust.go @@ -121,7 +121,8 @@ func fileSuppliedCredential(connection map[string]interface{}) bool { return false } value, ok := connection[key].(string) - return ok && value != "" && !isReference(value) + // Blank keys count as absent: executors that trim fall back to ambient credentials. + return ok && strings.TrimSpace(value) != "" && !isReference(value) } // fileControlledDestination returns the effective destination if the file chose it. diff --git a/runtime/go/prompty/model/vector_conformance_test.go b/runtime/go/prompty/model/vector_conformance_test.go index 7aa64abb6..f4444e0e4 100644 --- a/runtime/go/prompty/model/vector_conformance_test.go +++ b/runtime/go/prompty/model/vector_conformance_test.go @@ -2071,7 +2071,38 @@ func TestVector57LoadConformanceLoadUntrustedEndpointBackslashRejected(t *testin vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testing.T) { +func TestVector58LoadConformanceLoadUntrustedEndpointBlankLiteralKeyRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_blank_literal_key_rejected", + "description": "a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-blank-key", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": " " + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector59LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_env_default_rejected", "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", @@ -2101,7 +2132,7 @@ func TestVector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testi vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T) { +func TestVector60LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_env_key_rejected", "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", @@ -2134,7 +2165,7 @@ func TestVector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *testing.T) { +func TestVector61LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_file_connection_rejected", "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", @@ -2166,7 +2197,7 @@ func TestVector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected(t *testing.T) { +func TestVector62LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_fragment_before_at_rejected", "description": "A '#' ends the authority, so the host is evil.example.com", @@ -2195,7 +2226,7 @@ func TestVector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected(t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected(t *testing.T) { +func TestVector63LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_ipv6_literal_rejected", "description": "An IPv6 literal is not a trusted origin", @@ -2224,7 +2255,7 @@ func TestVector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected(t *test vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected(t *testing.T) { +func TestVector64LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_key_with_client_secret_rejected", "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", @@ -2254,7 +2285,7 @@ func TestVector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected(t *testing.T) { +func TestVector65LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_mismatched_credential_rejected", "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", @@ -2284,7 +2315,7 @@ func TestVector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejecte vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector65LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) { +func TestVector66LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_no_key_rejected", "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", @@ -2313,7 +2344,7 @@ func TestVector65LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected(t *testing.T) { +func TestVector67LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_oauth_literal_secret_rejected", "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", @@ -2346,7 +2377,7 @@ func TestVector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected( vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected(t *testing.T) { +func TestVector68LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_percent_encoded_host_rejected", "description": "Hosts are compared without percent-decoding, so an encoded host never matches", @@ -2375,7 +2406,7 @@ func TestVector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected( vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected(t *testing.T) { +func TestVector69LoadConformanceLoadUntrustedEndpointPortMismatchRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_port_mismatch_rejected", "description": "A non-default port is a different origin", @@ -2404,7 +2435,7 @@ func TestVector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected(t *tes vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected(t *testing.T) { +func TestVector70LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_query_before_at_rejected", "description": "A '?' ends the authority, so the host is evil.example.com", @@ -2433,7 +2464,7 @@ func TestVector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected(t *te vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected(t *testing.T) { +func TestVector71LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_scheme_mismatch_rejected", "description": "Built-in origins are https; http on the same host is a different origin", @@ -2462,7 +2493,7 @@ func TestVector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected(t *t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected(t *testing.T) { +func TestVector72LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_suffix_lookalike_rejected", "description": "A trusted host as a label prefix of another domain is not trusted", @@ -2491,7 +2522,7 @@ func TestVector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected(t * vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *testing.T) { +func TestVector73LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_suffix_spoof_rejected", "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", @@ -2523,7 +2554,7 @@ func TestVector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *test vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected(t *testing.T) { +func TestVector74LoadConformanceLoadUntrustedEndpointTrailingDotRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_trailing_dot_rejected", "description": "Hosts are compared exactly; a trailing-dot host does not match", @@ -2552,7 +2583,7 @@ func TestVector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected(t *test vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected(t *testing.T) { +func TestVector75LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_unicode_homoglyph_rejected", "description": "A non-ASCII lookalike host does not match", @@ -2581,7 +2612,7 @@ func TestVector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected(t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected(t *testing.T) { +func TestVector76LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_userinfo_lookalike_rejected", "description": "A trusted host in userinfo is not the destination host", @@ -2610,7 +2641,7 @@ func TestVector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected(t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *testing.T) { +func TestVector77LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_wildcard_apex_rejected", "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", @@ -2639,7 +2670,7 @@ func TestVector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *tes vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected(t *testing.T) { +func TestVector78LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected(t *testing.T) { vectorJSON := `{ "name": "untrusted_endpoint_wildcard_boundary_rejected", "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", @@ -2668,7 +2699,7 @@ func TestVector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected(t vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector78MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { +func TestVector79MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorJSON := `{ "name": "clear_by_category_and_all", "description": "Clear removes only the requested category, or all memories when no category is supplied.", @@ -2721,7 +2752,7 @@ func TestVector78MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector79MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { +func TestVector80MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { vectorJSON := `{ "name": "format_core_and_recall_results", "description": "Formatting injects only core memories into system prompts and formats recall results with category and tags.", @@ -2762,7 +2793,7 @@ func TestVector79MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector80MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { +func TestVector81MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { vectorJSON := `{ "name": "recall_empty_query_and_limit", "description": "Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.", @@ -2816,7 +2847,7 @@ func TestVector80MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector81MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { +func TestVector82MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { vectorJSON := `{ "name": "recall_ranks_by_weighted_score", "description": "Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.", @@ -2874,7 +2905,7 @@ func TestVector81MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { +func TestVector83MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { vectorJSON := `{ "name": "recall_score_ties_preserve_insertion_order", "description": "Recall preserves insertion order for memories with identical scores.", @@ -2931,7 +2962,7 @@ func TestVector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector83MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { +func TestVector84MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorJSON := `{ "name": "recall_tags_and_core_boost", "description": "Recall gives tag matches higher weight than content matches and boosts matching core memories.", @@ -2991,7 +3022,7 @@ func TestVector83MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { +func TestVector85MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { vectorJSON := `{ "name": "recall_tokenization_punctuation_unicode_and_dedup", "description": "Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.", @@ -3051,7 +3082,7 @@ func TestVector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAnd vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector85MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { +func TestVector86MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_by_tags", "description": "Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.", @@ -3114,7 +3145,7 @@ func TestVector85MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { +func TestVector87MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_missing_and_empty_tags", "description": "Remember treats missing and empty tag lists as equivalent for core-memory deduplication.", @@ -3169,7 +3200,7 @@ func TestVector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector87MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { +func TestVector88MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { vectorJSON := `{ "name": "remember_evicts_archival_first", "description": "Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.", @@ -3229,7 +3260,7 @@ func TestVector87MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing. vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector88MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { +func TestVector89MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorJSON := `{ "name": "snapshot_port_round_trip", "description": "A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.", @@ -3268,7 +3299,7 @@ func TestVector88MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector89MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { +func TestVector90MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { vectorJSON := `{ "name": "update_index_out_of_range_error", "description": "Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.", @@ -3301,7 +3332,7 @@ func TestVector89MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector90TurnConformanceReplayMaxIterations(t *testing.T) { +func TestVector91TurnConformanceReplayMaxIterations(t *testing.T) { vectorJSON := `{ "name": "max_iterations", "stage": "replay", @@ -3337,7 +3368,7 @@ func TestVector90TurnConformanceReplayMaxIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector91TurnConformanceReplayNoTool(t *testing.T) { +func TestVector92TurnConformanceReplayNoTool(t *testing.T) { vectorJSON := `{ "name": "no_tool", "stage": "replay", @@ -3369,7 +3400,7 @@ func TestVector91TurnConformanceReplayNoTool(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector92TurnConformanceReplayPermissionDenied(t *testing.T) { +func TestVector93TurnConformanceReplayPermissionDenied(t *testing.T) { vectorJSON := `{ "name": "permission_denied", "stage": "replay", @@ -3404,7 +3435,7 @@ func TestVector92TurnConformanceReplayPermissionDenied(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector93TurnConformanceReplayToolFailure(t *testing.T) { +func TestVector94TurnConformanceReplayToolFailure(t *testing.T) { vectorJSON := `{ "name": "tool_failure", "stage": "replay", @@ -3441,7 +3472,7 @@ func TestVector93TurnConformanceReplayToolFailure(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector94TurnConformanceReplayToolSuccess(t *testing.T) { +func TestVector95TurnConformanceReplayToolSuccess(t *testing.T) { vectorJSON := `{ "name": "tool_success", "stage": "replay", @@ -3478,7 +3509,7 @@ func TestVector94TurnConformanceReplayToolSuccess(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector95TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { +func TestVector96TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorJSON := `{ "name": "assistant_tool_calls_metadata", "description": "When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.", @@ -3631,7 +3662,7 @@ func TestVector95TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector96TurnConformanceRunAsyncToolFunction(t *testing.T) { +func TestVector97TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorJSON := `{ "name": "async_tool_function", "description": "Agent loop correctly handles async tool functions", @@ -3742,7 +3773,7 @@ func TestVector96TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector97TurnConformanceRunBindingsInjected(t *testing.T) { +func TestVector98TurnConformanceRunBindingsInjected(t *testing.T) { vectorJSON := `{ "name": "bindings_injected", "description": "Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.", @@ -3873,7 +3904,7 @@ func TestVector97TurnConformanceRunBindingsInjected(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector98TurnConformanceRunCancellationBeforeLlm(t *testing.T) { +func TestVector99TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorJSON := `{ "name": "cancellation_before_llm", "description": "§13.2 Cancellation — Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.", @@ -3935,7 +3966,7 @@ func TestVector98TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector99TurnConformanceRunCancellationBetweenIterations(t *testing.T) { +func TestVector100TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_iterations", "description": "§13.2 Cancellation — Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.", @@ -4066,7 +4097,7 @@ func TestVector99TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector100TurnConformanceRunCancellationBetweenTools(t *testing.T) { +func TestVector101TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_tools", "description": "§13.2 Cancellation — LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.", @@ -4206,7 +4237,7 @@ func TestVector100TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector101TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { +func TestVector102TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorJSON := `{ "name": "context_no_trim_when_fits", "description": "§13.3 Context Window — Messages fit within the context budget. No trimming occurs and all messages are preserved.", @@ -4278,7 +4309,7 @@ func TestVector101TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector102TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { +func TestVector103TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { vectorJSON := `{ "name": "context_preserves_system_messages", "description": "§13.3 Context Window — Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.", @@ -4386,7 +4417,7 @@ func TestVector102TurnConformanceRunContextPreservesSystemMessages(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector103TurnConformanceRunContextTrimBasic(t *testing.T) { +func TestVector104TurnConformanceRunContextTrimBasic(t *testing.T) { vectorJSON := `{ "name": "context_trim_basic", "description": "§13.3 Context Window — Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.", @@ -4545,7 +4576,7 @@ func TestVector103TurnConformanceRunContextTrimBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector104TurnConformanceRunEmptyToolResult(t *testing.T) { +func TestVector105TurnConformanceRunEmptyToolResult(t *testing.T) { vectorJSON := `{ "name": "empty_tool_result", "description": "Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content — it must not be skipped or filtered out.", @@ -4661,7 +4692,7 @@ func TestVector104TurnConformanceRunEmptyToolResult(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector105TurnConformanceRunEventsBasicToolLoop(t *testing.T) { +func TestVector106TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorJSON := `{ "name": "events_basic_tool_loop", "description": "§13.1 Events — A 2-turn tool-call loop emits the full event lifecycle: status → tool_call_start → tool_result → messages_updated → done.", @@ -4808,7 +4839,7 @@ func TestVector105TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector106TurnConformanceRunEventsErrorLogged(t *testing.T) { +func TestVector107TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorJSON := `{ "name": "events_error_logged", "description": "§13.1 Events — A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.", @@ -4935,7 +4966,7 @@ func TestVector106TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector107TurnConformanceRunEventsNoTools(t *testing.T) { +func TestVector108TurnConformanceRunEventsNoTools(t *testing.T) { vectorJSON := `{ "name": "events_no_tools", "description": "§13.1 Events — Single-turn completion with no tool calls. Only the done event is emitted.", @@ -5014,7 +5045,7 @@ func TestVector107TurnConformanceRunEventsNoTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector108TurnConformanceRunGuardrailAllPass(t *testing.T) { +func TestVector109TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorJSON := `{ "name": "guardrail_all_pass", "description": "§13.4 Guardrails — All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.", @@ -5138,7 +5169,7 @@ func TestVector108TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector109TurnConformanceRunGuardrailInputDeny(t *testing.T) { +func TestVector110TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_input_deny", "description": "§13.4 Guardrails — Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.", @@ -5195,7 +5226,7 @@ func TestVector109TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector110TurnConformanceRunGuardrailOutputDeny(t *testing.T) { +func TestVector111TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_output_deny", "description": "§13.4 Guardrails — Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.", @@ -5274,7 +5305,7 @@ func TestVector110TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector111TurnConformanceRunGuardrailToolDeny(t *testing.T) { +func TestVector112TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_tool_deny", "description": "§13.4 Guardrails — LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.", @@ -5429,7 +5460,7 @@ func TestVector111TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector112TurnConformanceRunMaxIterationsExceeded(t *testing.T) { +func TestVector113TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorJSON := `{ "name": "max_iterations_exceeded", "description": "LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.", @@ -5940,7 +5971,7 @@ func TestVector112TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector113TurnConformanceRunMultiTurnToolCalls(t *testing.T) { +func TestVector114TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorJSON := `{ "name": "multi_turn_tool_calls", "description": "Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.", @@ -6118,7 +6149,7 @@ func TestVector113TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector114TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { +func TestVector115TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorJSON := `{ "name": "multiple_tool_calls_single_turn", "description": "Turn 1: LLM returns two parallel tool calls — get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.", @@ -6300,7 +6331,7 @@ func TestVector114TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector115TurnConformanceRunNoToolCalls(t *testing.T) { +func TestVector116TurnConformanceRunNoToolCalls(t *testing.T) { vectorJSON := `{ "name": "no_tool_calls", "description": "LLM returns content immediately with no tool calls — the agent loop completes in a single iteration.", @@ -6369,7 +6400,7 @@ func TestVector115TurnConformanceRunNoToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector116TurnConformanceRunParallelToolsBasic(t *testing.T) { +func TestVector117TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_basic", "description": "§13.6 Parallel Tools — LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.", @@ -6543,7 +6574,7 @@ func TestVector116TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector117TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { +func TestVector118TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_guardrail_deny", "description": "§13.6 Parallel Tools + §13.4 Guardrails — 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.", @@ -6735,7 +6766,7 @@ func TestVector117TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector118TurnConformanceRunParallelToolsWithToolError(t *testing.T) { +func TestVector119TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_tool_error", "description": "§13.6 Parallel Tools + tool error — 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.", @@ -6909,7 +6940,7 @@ func TestVector118TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector119TurnConformanceRunSingleToolCall(t *testing.T) { +func TestVector120TurnConformanceRunSingleToolCall(t *testing.T) { vectorJSON := `{ "name": "single_tool_call", "description": "Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.", @@ -7057,7 +7088,7 @@ func TestVector119TurnConformanceRunSingleToolCall(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector120TurnConformanceRunSteeringInjectMessage(t *testing.T) { +func TestVector121TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorJSON := `{ "name": "steering_inject_message", "description": "§13.5 Steering — After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.", @@ -7266,7 +7297,7 @@ func TestVector120TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector121TurnConformanceRunSteeringMultipleMessages(t *testing.T) { +func TestVector122TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorJSON := `{ "name": "steering_multiple_messages", "description": "§13.5 Steering — Two steering messages are queued and both injected before iteration 2. Tests atomic drain — both appear in order.", @@ -7437,7 +7468,7 @@ func TestVector121TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector122TurnConformanceRunToolNotRegisteredError(t *testing.T) { +func TestVector123TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorJSON := `{ "name": "tool_not_registered_error", "description": "LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.", @@ -7522,7 +7553,7 @@ func TestVector122TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector123TurnConformanceRunToolResultMessageFormat(t *testing.T) { +func TestVector124TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorJSON := `{ "name": "tool_result_message_format", "description": "After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.", @@ -7644,7 +7675,7 @@ func TestVector123TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector124TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { +func TestVector125TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorJSON := `{ "name": "cancel_before_context", "stage": "turn", @@ -7677,7 +7708,7 @@ func TestVector124TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector125TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { +func TestVector126TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorJSON := `{ "name": "delegated_provider_state", "stage": "turn", @@ -7745,7 +7776,7 @@ func TestVector125TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector126TurnConformanceRunTurnFinalOutput(t *testing.T) { +func TestVector127TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorJSON := `{ "name": "final_output", "stage": "turn", @@ -7791,7 +7822,7 @@ func TestVector126TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector127TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { +func TestVector128TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "memory_context_is_model_visible", "stage": "turn", @@ -7854,7 +7885,7 @@ func TestVector127TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector128TurnConformanceRunTurnOrderedToolRound(t *testing.T) { +func TestVector129TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorJSON := `{ "name": "ordered_tool_round", "stage": "turn", @@ -7946,7 +7977,7 @@ func TestVector128TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector129TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { +func TestVector130TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "permission_denial_is_model_visible", "stage": "turn", @@ -7998,7 +8029,7 @@ func TestVector129TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testin vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector130WireConformanceToRequestAnthropicImageFormat(t *testing.T) { +func TestVector131WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorJSON := `{ "name": "anthropic_image_format", "description": "§7.5 — Anthropic: ImagePart uses base64 source block with media_type, not image_url format.", @@ -8069,7 +8100,7 @@ func TestVector130WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector131WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { +func TestVector132WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { vectorJSON := `{ "name": "anthropic_max_tokens_required", "description": "§7.5 — Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).", @@ -8125,7 +8156,7 @@ func TestVector131WireConformanceToRequestAnthropicMaxTokensRequired(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector132WireConformanceToRequestAnthropicOptions(t *testing.T) { +func TestVector133WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorJSON := `{ "name": "anthropic_options", "description": "Anthropic: option names map correctly (topK, stopSequences)", @@ -8195,7 +8226,7 @@ func TestVector132WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector133WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { +func TestVector134WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { vectorJSON := `{ "name": "anthropic_system_separate", "description": "§7.5 — Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.", @@ -8263,7 +8294,7 @@ func TestVector133WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector134WireConformanceToRequestAnthropicToolWire(t *testing.T) { +func TestVector135WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorJSON := `{ "name": "anthropic_tool_wire", "description": "Anthropic: tools use input_schema (not parameters), name at top level", @@ -8349,7 +8380,7 @@ func TestVector134WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector135WireConformanceToRequestChatAudioMp3(t *testing.T) { +func TestVector136WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorJSON := `{ "name": "chat_audio_mp3", "description": "§7.1.2 — AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.", @@ -8408,7 +8439,7 @@ func TestVector135WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector136WireConformanceToRequestChatAudioPart(t *testing.T) { +func TestVector137WireConformanceToRequestChatAudioPart(t *testing.T) { vectorJSON := `{ "name": "chat_audio_part", "description": "§7.1.2 — AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.", @@ -8467,7 +8498,7 @@ func TestVector136WireConformanceToRequestChatAudioPart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector137WireConformanceToRequestChatImageBase64(t *testing.T) { +func TestVector138WireConformanceToRequestChatImageBase64(t *testing.T) { vectorJSON := `{ "name": "chat_image_base64", "description": "Base64 data URI image part preserved as-is in image_url", @@ -8532,7 +8563,7 @@ func TestVector137WireConformanceToRequestChatImageBase64(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector138WireConformanceToRequestChatImagePart(t *testing.T) { +func TestVector139WireConformanceToRequestChatImagePart(t *testing.T) { vectorJSON := `{ "name": "chat_image_part", "description": "Image content part converted to OpenAI image_url format", @@ -8597,7 +8628,7 @@ func TestVector138WireConformanceToRequestChatImagePart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector139WireConformanceToRequestChatMultipartContent(t *testing.T) { +func TestVector140WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorJSON := `{ "name": "chat_multipart_content", "description": "§7.1.1/§7.1.2 — Message with TextPart + ImagePart MUST produce an array of typed content blocks.", @@ -8662,7 +8693,7 @@ func TestVector139WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector140WireConformanceToRequestChatSimple(t *testing.T) { +func TestVector141WireConformanceToRequestChatSimple(t *testing.T) { vectorJSON := `{ "name": "chat_simple", "description": "§7.1.7 — Two messages (system + user), no tools, no options. Minimal chat request.", @@ -8725,7 +8756,7 @@ func TestVector140WireConformanceToRequestChatSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector141WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { +func TestVector142WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { vectorJSON := `{ "name": "chat_single_text_optimized", "description": "§7.1.1 — Message with exactly 1 TextPart MUST produce a plain string content, not an array.", @@ -8775,7 +8806,7 @@ func TestVector141WireConformanceToRequestChatSingleTextOptimized(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector142WireConformanceToRequestChatWithOptions(t *testing.T) { +func TestVector143WireConformanceToRequestChatWithOptions(t *testing.T) { vectorJSON := `{ "name": "chat_with_options", "description": "§7.1.5 — All standard ModelOptions mapped to OpenAI parameter names.", @@ -8844,7 +8875,7 @@ func TestVector142WireConformanceToRequestChatWithOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector143WireConformanceToRequestEmbeddingWire(t *testing.T) { +func TestVector144WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorJSON := `{ "name": "embedding_wire", "description": "§7.2 — Embedding request: text extracted from messages, single string input (not array).", @@ -8889,7 +8920,7 @@ func TestVector143WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector144WireConformanceToRequestImageWire(t *testing.T) { +func TestVector145WireConformanceToRequestImageWire(t *testing.T) { vectorJSON := `{ "name": "image_wire", "description": "§7.3 — Image generation request: prompt extracted from last user message.", @@ -8934,7 +8965,7 @@ func TestVector144WireConformanceToRequestImageWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector145WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { +func TestVector146WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorJSON := `{ "name": "kind_to_json_type_mapping", "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.", @@ -9054,7 +9085,7 @@ func TestVector145WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector146WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { +func TestVector147WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { vectorJSON := `{ "name": "options_additional_properties", "description": "§7.1.5 — additionalProperties from ModelOptions MUST be merged into the request as top-level keys.", @@ -9109,7 +9140,7 @@ func TestVector146WireConformanceToRequestOptionsAdditionalProperties(t *testing vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector147WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { +func TestVector148WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { vectorJSON := `{ "name": "options_max_completion_tokens", "description": "§7.1.5 — maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.", @@ -9162,7 +9193,7 @@ func TestVector147WireConformanceToRequestOptionsMaxCompletionTokens(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector148WireConformanceToRequestOptionsStopSequences(t *testing.T) { +func TestVector149WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorJSON := `{ "name": "options_stop_sequences", "description": "§7.1.5 — stopSequences MUST map to 'stop' in the OpenAI wire format.", @@ -9221,7 +9252,7 @@ func TestVector148WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector149WireConformanceToRequestResponsesSimple(t *testing.T) { +func TestVector150WireConformanceToRequestResponsesSimple(t *testing.T) { vectorJSON := `{ "name": "responses_simple", "description": "Responses API: system becomes instructions, messages become input items", @@ -9281,7 +9312,7 @@ func TestVector149WireConformanceToRequestResponsesSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector150WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { +func TestVector151WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "responses_structured_output", "description": "Responses API: structured output uses text.format.json_schema", @@ -9369,7 +9400,7 @@ func TestVector150WireConformanceToRequestResponsesStructuredOutput(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector151WireConformanceToRequestResponsesWithTools(t *testing.T) { +func TestVector152WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorJSON := `{ "name": "responses_with_tools", "description": "Responses API: tools use flat format (no nested function key)", @@ -9450,7 +9481,7 @@ func TestVector151WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector152WireConformanceToRequestStructuredOutput(t *testing.T) { +func TestVector153WireConformanceToRequestStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "structured_output", "description": "§7.1.6 — outputs converted to response_format with json_schema, strict=true, additionalProperties=false.", @@ -9534,7 +9565,7 @@ func TestVector152WireConformanceToRequestStructuredOutput(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector153WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { +func TestVector154WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { vectorJSON := `{ "name": "structured_output_nested_optional", "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", @@ -9637,7 +9668,7 @@ func TestVector153WireConformanceToRequestStructuredOutputNestedOptional(t *test vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector154WireConformanceToRequestToolsBindingsStripped(t *testing.T) { +func TestVector155WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorJSON := `{ "name": "tools_bindings_stripped", "description": "§7.1.3 — Parameters listed in bindings MUST be stripped from wire tools (properties AND required).", @@ -9730,7 +9761,7 @@ func TestVector154WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector155WireConformanceToRequestToolsFunctionWire(t *testing.T) { +func TestVector156WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorJSON := `{ "name": "tools_function_wire", "description": "§7.1.3 — FunctionTool projected as OpenAI function definition with JSON Schema parameters.", @@ -9813,7 +9844,7 @@ func TestVector155WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector156WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { +func TestVector157WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorJSON := `{ "name": "tools_null_when_empty", "description": "§7.1.3 — When no tools are defined, the tools key MUST be absent from the request entirely.", @@ -9863,7 +9894,7 @@ func TestVector156WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector157WireConformanceToRequestToolsStrictMode(t *testing.T) { +func TestVector158WireConformanceToRequestToolsStrictMode(t *testing.T) { vectorJSON := `{ "name": "tools_strict_mode", "description": "§7.1.3 — FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.", diff --git a/runtime/go/prompty/model/vectorconformance/vector_conformance.go b/runtime/go/prompty/model/vectorconformance/vector_conformance.go index db81ac528..c66decc53 100644 --- a/runtime/go/prompty/model/vectorconformance/vector_conformance.go +++ b/runtime/go/prompty/model/vectorconformance/vector_conformance.go @@ -1317,6 +1317,17 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("untrusted_endpoint_backslash_rejected: expected an error, got nil") } } + // vector: untrusted_endpoint_blank_literal_key_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_blank_literal_key_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_blank_literal_key_rejected: expected an error, got nil") + } + } // vector: untrusted_endpoint_env_default_rejected { var source interface{} diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java index 0116b710c..97d675b9d 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java @@ -213,7 +213,8 @@ private static boolean fileSuppliedCredential(Map connection) { if (key == null) { return false; } - return connection.get(key) instanceof String value && !value.isEmpty() && !isReference(value); + // Blank keys count as absent: executors that trim fall back to ambient credentials. + return connection.get(key) instanceof String value && !value.isBlank() && !isReference(value); } /** The effective destination if the file chose it, or {@code null} when the host did. */ diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java index 513eae313..261bf926d 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java @@ -732,6 +732,20 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex } if (caught == null) throw new AssertionError("untrusted_endpoint_backslash_rejected: expected an error"); } + { + // vector: untrusted_endpoint_blank_literal_key_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-blank-key\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\" \"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_blank_literal_key_rejected: expected an error"); + } { // vector: untrusted_endpoint_env_default_rejected StringBuilder inputJson = new StringBuilder(); diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java index 8591718e1..72d2084a6 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java @@ -463,700 +463,707 @@ private static void vector57LoadConformanceLoadUntrustedEndpointBackslashRejecte VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() throws Exception { + private static void vector58LoadConformanceLoadUntrustedEndpointBlankLiteralKeyRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_blank_literal_key_rejected\",\"description\":\"a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-blank-key\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\" \"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector59LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected() throws Exception { + private static void vector60LoadConformanceLoadUntrustedEndpointEnvKeyRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected() throws Exception { + private static void vector61LoadConformanceLoadUntrustedEndpointFileConnectionRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() throws Exception { + private static void vector62LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_fragment_before_at_rejected\",\"description\":\"A '#' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() throws Exception { + private static void vector63LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_ipv6_literal_rejected\",\"description\":\"An IPv6 literal is not a trusted origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() throws Exception { + private static void vector64LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() throws Exception { + private static void vector65LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector65LoadConformanceLoadUntrustedEndpointNoKeyRejected() throws Exception { + private static void vector66LoadConformanceLoadUntrustedEndpointNoKeyRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() throws Exception { + private static void vector67LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_oauth_literal_secret_rejected\",\"description\":\"kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() throws Exception { + private static void vector68LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_percent_encoded_host_rejected\",\"description\":\"Hosts are compared without percent-decoding, so an encoded host never matches\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected() throws Exception { + private static void vector69LoadConformanceLoadUntrustedEndpointPortMismatchRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_port_mismatch_rejected\",\"description\":\"A non-default port is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() throws Exception { + private static void vector70LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_query_before_at_rejected\",\"description\":\"A '?' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() throws Exception { + private static void vector71LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_scheme_mismatch_rejected\",\"description\":\"Built-in origins are https; http on the same host is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() throws Exception { + private static void vector72LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_suffix_lookalike_rejected\",\"description\":\"A trusted host as a label prefix of another domain is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() throws Exception { + private static void vector73LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected() throws Exception { + private static void vector74LoadConformanceLoadUntrustedEndpointTrailingDotRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_trailing_dot_rejected\",\"description\":\"Hosts are compared exactly; a trailing-dot host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() throws Exception { + private static void vector75LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_unicode_homoglyph_rejected\",\"description\":\"A non-ASCII lookalike host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\\u0430pi.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() throws Exception { + private static void vector76LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_userinfo_lookalike_rejected\",\"description\":\"A trusted host in userinfo is not the destination host\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected() throws Exception { + private static void vector77LoadConformanceLoadUntrustedEndpointWildcardApexRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() throws Exception { + private static void vector78LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"untrusted_endpoint_wildcard_boundary_rejected\",\"description\":\"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector78MemoryConformanceOperateClearByCategoryAndAll() throws Exception { + private static void vector79MemoryConformanceOperateClearByCategoryAndAll() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector79MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { + private static void vector80MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector80MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { + private static void vector81MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector81MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { + private static void vector82MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { + private static void vector83MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector83MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { + private static void vector84MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { + private static void vector85MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\\u00e9\"]},{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\\u00e9\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector85MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { + private static void vector86MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { + private static void vector87MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector87MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { + private static void vector88MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector88MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { + private static void vector89MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector89MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { + private static void vector90MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector90TurnConformanceReplayMaxIterations() throws Exception { + private static void vector91TurnConformanceReplayMaxIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector91TurnConformanceReplayNoTool() throws Exception { + private static void vector92TurnConformanceReplayNoTool() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector92TurnConformanceReplayPermissionDenied() throws Exception { + private static void vector93TurnConformanceReplayPermissionDenied() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector93TurnConformanceReplayToolFailure() throws Exception { + private static void vector94TurnConformanceReplayToolFailure() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector94TurnConformanceReplayToolSuccess() throws Exception { + private static void vector95TurnConformanceReplayToolSuccess() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector95TurnConformanceRunAssistantToolCallsMetadata() throws Exception { + private static void vector96TurnConformanceRunAssistantToolCallsMetadata() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector96TurnConformanceRunAsyncToolFunction() throws Exception { + private static void vector97TurnConformanceRunAsyncToolFunction() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector97TurnConformanceRunBindingsInjected() throws Exception { + private static void vector98TurnConformanceRunBindingsInjected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\\u00b0C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector98TurnConformanceRunCancellationBeforeLlm() throws Exception { + private static void vector99TurnConformanceRunCancellationBeforeLlm() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_before_llm\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector99TurnConformanceRunCancellationBetweenIterations() throws Exception { + private static void vector100TurnConformanceRunCancellationBetweenIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_iterations\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector100TurnConformanceRunCancellationBetweenTools() throws Exception { + private static void vector101TurnConformanceRunCancellationBetweenTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_tools\",\"description\":\"\\u00a713.2 Cancellation \\u2014 LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector101TurnConformanceRunContextNoTrimWhenFits() throws Exception { + private static void vector102TurnConformanceRunContextNoTrimWhenFits() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector102TurnConformanceRunContextPreservesSystemMessages() throws Exception { + private static void vector103TurnConformanceRunContextPreservesSystemMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_preserves_system_messages\",\"description\":\"\\u00a713.3 Context Window \\u2014 Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \\u2014 they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector103TurnConformanceRunContextTrimBasic() throws Exception { + private static void vector104TurnConformanceRunContextTrimBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_trim_basic\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector104TurnConformanceRunEmptyToolResult() throws Exception { + private static void vector105TurnConformanceRunEmptyToolResult() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \\u2014 it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector105TurnConformanceRunEventsBasicToolLoop() throws Exception { + private static void vector106TurnConformanceRunEventsBasicToolLoop() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_basic_tool_loop\",\"description\":\"\\u00a713.1 Events \\u2014 A 2-turn tool-call loop emits the full event lifecycle: status \\u2192 tool_call_start \\u2192 tool_result \\u2192 messages_updated \\u2192 done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\\u00b0F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector106TurnConformanceRunEventsErrorLogged() throws Exception { + private static void vector107TurnConformanceRunEventsErrorLogged() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_error_logged\",\"description\":\"\\u00a713.1 Events \\u0097 A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \\u0097 this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector107TurnConformanceRunEventsNoTools() throws Exception { + private static void vector108TurnConformanceRunEventsNoTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_no_tools\",\"description\":\"\\u00a713.1 Events \\u2014 Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector108TurnConformanceRunGuardrailAllPass() throws Exception { + private static void vector109TurnConformanceRunGuardrailAllPass() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_all_pass\",\"description\":\"\\u00a713.4 Guardrails \\u2014 All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector109TurnConformanceRunGuardrailInputDeny() throws Exception { + private static void vector110TurnConformanceRunGuardrailInputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_input_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector110TurnConformanceRunGuardrailOutputDeny() throws Exception { + private static void vector111TurnConformanceRunGuardrailOutputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_output_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector111TurnConformanceRunGuardrailToolDeny() throws Exception { + private static void vector112TurnConformanceRunGuardrailToolDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_tool_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \\u2014 its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector112TurnConformanceRunMaxIterationsExceeded() throws Exception { + private static void vector113TurnConformanceRunMaxIterationsExceeded() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\\u00b0F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\\u00b0F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\\u00b0F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\\u00b0F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\\u00b0F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\\u00b0F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\\u00b0F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\\u00b0F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\\u00b0F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector113TurnConformanceRunMultiTurnToolCalls() throws Exception { + private static void vector114TurnConformanceRunMultiTurnToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\\u00b0C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector114TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { + private static void vector115TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \\u2014 get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\\u00b0F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector115TurnConformanceRunNoToolCalls() throws Exception { + private static void vector116TurnConformanceRunNoToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \\u2014 the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector116TurnConformanceRunParallelToolsBasic() throws Exception { + private static void vector117TurnConformanceRunParallelToolsBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_basic\",\"description\":\"\\u00a713.6 Parallel Tools \\u2014 LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector117TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { + private static void vector118TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\\u00a713.6 Parallel Tools + \\u00a713.4 Guardrails \\u2014 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector118TurnConformanceRunParallelToolsWithToolError() throws Exception { + private static void vector119TurnConformanceRunParallelToolsWithToolError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\\u00a713.6 Parallel Tools + tool error \\u2014 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector119TurnConformanceRunSingleToolCall() throws Exception { + private static void vector120TurnConformanceRunSingleToolCall() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector120TurnConformanceRunSteeringInjectMessage() throws Exception { + private static void vector121TurnConformanceRunSteeringInjectMessage() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_inject_message\",\"description\":\"\\u00a713.5 Steering \\u2014 After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector121TurnConformanceRunSteeringMultipleMessages() throws Exception { + private static void vector122TurnConformanceRunSteeringMultipleMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_multiple_messages\",\"description\":\"\\u00a713.5 Steering \\u2014 Two steering messages are queued and both injected before iteration 2. Tests atomic drain \\u2014 both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\\u00b0F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector122TurnConformanceRunToolNotRegisteredError() throws Exception { + private static void vector123TurnConformanceRunToolNotRegisteredError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector123TurnConformanceRunToolResultMessageFormat() throws Exception { + private static void vector124TurnConformanceRunToolResultMessageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\\u00b0F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\\u00b0F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector124TurnConformanceRunTurnCancelBeforeContext() throws Exception { + private static void vector125TurnConformanceRunTurnCancelBeforeContext() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector125TurnConformanceRunTurnDelegatedProviderState() throws Exception { + private static void vector126TurnConformanceRunTurnDelegatedProviderState() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector126TurnConformanceRunTurnFinalOutput() throws Exception { + private static void vector127TurnConformanceRunTurnFinalOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector127TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { + private static void vector128TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector128TurnConformanceRunTurnOrderedToolRound() throws Exception { + private static void vector129TurnConformanceRunTurnOrderedToolRound() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector129TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { + private static void vector130TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector130WireConformanceToRequestAnthropicImageFormat() throws Exception { + private static void vector131WireConformanceToRequestAnthropicImageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_image_format\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector131WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { + private static void vector132WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector132WireConformanceToRequestAnthropicOptions() throws Exception { + private static void vector133WireConformanceToRequestAnthropicOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector133WireConformanceToRequestAnthropicSystemSeparate() throws Exception { + private static void vector134WireConformanceToRequestAnthropicSystemSeparate() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_system_separate\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector134WireConformanceToRequestAnthropicToolWire() throws Exception { + private static void vector135WireConformanceToRequestAnthropicToolWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector135WireConformanceToRequestChatAudioMp3() throws Exception { + private static void vector136WireConformanceToRequestChatAudioMp3() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_mp3\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector136WireConformanceToRequestChatAudioPart() throws Exception { + private static void vector137WireConformanceToRequestChatAudioPart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_part\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector137WireConformanceToRequestChatImageBase64() throws Exception { + private static void vector138WireConformanceToRequestChatImageBase64() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector138WireConformanceToRequestChatImagePart() throws Exception { + private static void vector139WireConformanceToRequestChatImagePart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector139WireConformanceToRequestChatMultipartContent() throws Exception { + private static void vector140WireConformanceToRequestChatMultipartContent() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_multipart_content\",\"description\":\"\\u00a77.1.1/\\u00a77.1.2 \\u2014 Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector140WireConformanceToRequestChatSimple() throws Exception { + private static void vector141WireConformanceToRequestChatSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_simple\",\"description\":\"\\u00a77.1.7 \\u2014 Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector141WireConformanceToRequestChatSingleTextOptimized() throws Exception { + private static void vector142WireConformanceToRequestChatSingleTextOptimized() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_single_text_optimized\",\"description\":\"\\u00a77.1.1 \\u2014 Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector142WireConformanceToRequestChatWithOptions() throws Exception { + private static void vector143WireConformanceToRequestChatWithOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_with_options\",\"description\":\"\\u00a77.1.5 \\u2014 All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector143WireConformanceToRequestEmbeddingWire() throws Exception { + private static void vector144WireConformanceToRequestEmbeddingWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"embedding_wire\",\"description\":\"\\u00a77.2 \\u2014 Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector144WireConformanceToRequestImageWire() throws Exception { + private static void vector145WireConformanceToRequestImageWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"image_wire\",\"description\":\"\\u00a77.3 \\u2014 Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector145WireConformanceToRequestKindToJsonTypeMapping() throws Exception { + private static void vector146WireConformanceToRequestKindToJsonTypeMapping() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\\u00a77.1.4 \\u2014 All Property kind values mapped to JSON Schema types: string\\u2192string, integer\\u2192integer, float\\u2192number, boolean\\u2192boolean, array\\u2192array, object\\u2192object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector146WireConformanceToRequestOptionsAdditionalProperties() throws Exception { + private static void vector147WireConformanceToRequestOptionsAdditionalProperties() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_additional_properties\",\"description\":\"\\u00a77.1.5 \\u2014 additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector147WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { + private static void vector148WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_max_completion_tokens\",\"description\":\"\\u00a77.1.5 \\u2014 maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector148WireConformanceToRequestOptionsStopSequences() throws Exception { + private static void vector149WireConformanceToRequestOptionsStopSequences() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_stop_sequences\",\"description\":\"\\u00a77.1.5 \\u2014 stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector149WireConformanceToRequestResponsesSimple() throws Exception { + private static void vector150WireConformanceToRequestResponsesSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector150WireConformanceToRequestResponsesStructuredOutput() throws Exception { + private static void vector151WireConformanceToRequestResponsesStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector151WireConformanceToRequestResponsesWithTools() throws Exception { + private static void vector152WireConformanceToRequestResponsesWithTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector152WireConformanceToRequestStructuredOutput() throws Exception { + private static void vector153WireConformanceToRequestStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output\",\"description\":\"\\u00a77.1.6 \\u2014 outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector153WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { + private static void vector154WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output_nested_optional\",\"description\":\"\\u00a77.1.4/\\u00a77.1.6 \\u2014 OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector154WireConformanceToRequestToolsBindingsStripped() throws Exception { + private static void vector155WireConformanceToRequestToolsBindingsStripped() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_bindings_stripped\",\"description\":\"\\u00a77.1.3 \\u2014 Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector155WireConformanceToRequestToolsFunctionWire() throws Exception { + private static void vector156WireConformanceToRequestToolsFunctionWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_function_wire\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector156WireConformanceToRequestToolsNullWhenEmpty() throws Exception { + private static void vector157WireConformanceToRequestToolsNullWhenEmpty() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_null_when_empty\",\"description\":\"\\u00a77.1.3 \\u2014 When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector157WireConformanceToRequestToolsStrictMode() throws Exception { + private static void vector158WireConformanceToRequestToolsStrictMode() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_strict_mode\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); @@ -1223,106 +1230,107 @@ public static void run() { failed += runCase("LoadConformance.load:trusted_endpoint_userinfo_allowed", VectorConformanceTests::vector55LoadConformanceLoadTrustedEndpointUserinfoAllowed); failed += runCase("LoadConformance.load:untrusted_endpoint_ambient_rejected", VectorConformanceTests::vector56LoadConformanceLoadUntrustedEndpointAmbientRejected); failed += runCase("LoadConformance.load:untrusted_endpoint_backslash_rejected", VectorConformanceTests::vector57LoadConformanceLoadUntrustedEndpointBackslashRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_env_default_rejected", VectorConformanceTests::vector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_env_key_rejected", VectorConformanceTests::vector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_file_connection_rejected", VectorConformanceTests::vector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_fragment_before_at_rejected", VectorConformanceTests::vector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_ipv6_literal_rejected", VectorConformanceTests::vector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", VectorConformanceTests::vector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", VectorConformanceTests::vector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_no_key_rejected", VectorConformanceTests::vector65LoadConformanceLoadUntrustedEndpointNoKeyRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_oauth_literal_secret_rejected", VectorConformanceTests::vector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_percent_encoded_host_rejected", VectorConformanceTests::vector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_port_mismatch_rejected", VectorConformanceTests::vector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_query_before_at_rejected", VectorConformanceTests::vector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_scheme_mismatch_rejected", VectorConformanceTests::vector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_lookalike_rejected", VectorConformanceTests::vector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", VectorConformanceTests::vector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_trailing_dot_rejected", VectorConformanceTests::vector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_unicode_homoglyph_rejected", VectorConformanceTests::vector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_userinfo_lookalike_rejected", VectorConformanceTests::vector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", VectorConformanceTests::vector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected); - failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_boundary_rejected", VectorConformanceTests::vector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected); - failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector78MemoryConformanceOperateClearByCategoryAndAll); - failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector79MemoryConformanceOperateFormatCoreAndRecallResults); - failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector80MemoryConformanceOperateRecallEmptyQueryAndLimit); - failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector81MemoryConformanceOperateRecallRanksByWeightedScore); - failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); - failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector83MemoryConformanceOperateRecallTagsAndCoreBoost); - failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); - failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector85MemoryConformanceOperateRememberDedupsCoreByTags); - failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); - failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector87MemoryConformanceOperateRememberEvictsArchivalFirst); - failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector88MemoryConformanceOperateSnapshotPortRoundTrip); - failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector89MemoryConformanceOperateUpdateIndexOutOfRangeError); - failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector90TurnConformanceReplayMaxIterations); - failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector91TurnConformanceReplayNoTool); - failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector92TurnConformanceReplayPermissionDenied); - failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector93TurnConformanceReplayToolFailure); - failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector94TurnConformanceReplayToolSuccess); - failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector95TurnConformanceRunAssistantToolCallsMetadata); - failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector96TurnConformanceRunAsyncToolFunction); - failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector97TurnConformanceRunBindingsInjected); - failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector98TurnConformanceRunCancellationBeforeLlm); - failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector99TurnConformanceRunCancellationBetweenIterations); - failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector100TurnConformanceRunCancellationBetweenTools); - failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector101TurnConformanceRunContextNoTrimWhenFits); - failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector102TurnConformanceRunContextPreservesSystemMessages); - failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector103TurnConformanceRunContextTrimBasic); - failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector104TurnConformanceRunEmptyToolResult); - failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector105TurnConformanceRunEventsBasicToolLoop); - failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector106TurnConformanceRunEventsErrorLogged); - failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector107TurnConformanceRunEventsNoTools); - failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector108TurnConformanceRunGuardrailAllPass); - failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector109TurnConformanceRunGuardrailInputDeny); - failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector110TurnConformanceRunGuardrailOutputDeny); - failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector111TurnConformanceRunGuardrailToolDeny); - failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector112TurnConformanceRunMaxIterationsExceeded); - failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector113TurnConformanceRunMultiTurnToolCalls); - failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector114TurnConformanceRunMultipleToolCallsSingleTurn); - failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector115TurnConformanceRunNoToolCalls); - failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector116TurnConformanceRunParallelToolsBasic); - failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector117TurnConformanceRunParallelToolsWithGuardrailDeny); - failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector118TurnConformanceRunParallelToolsWithToolError); - failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector119TurnConformanceRunSingleToolCall); - failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector120TurnConformanceRunSteeringInjectMessage); - failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector121TurnConformanceRunSteeringMultipleMessages); - failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector122TurnConformanceRunToolNotRegisteredError); - failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector123TurnConformanceRunToolResultMessageFormat); - failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector124TurnConformanceRunTurnCancelBeforeContext); - failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector125TurnConformanceRunTurnDelegatedProviderState); - failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector126TurnConformanceRunTurnFinalOutput); - failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector127TurnConformanceRunTurnMemoryContextIsModelVisible); - failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector128TurnConformanceRunTurnOrderedToolRound); - failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector129TurnConformanceRunTurnPermissionDenialIsModelVisible); - failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector130WireConformanceToRequestAnthropicImageFormat); - failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector131WireConformanceToRequestAnthropicMaxTokensRequired); - failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector132WireConformanceToRequestAnthropicOptions); - failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector133WireConformanceToRequestAnthropicSystemSeparate); - failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector134WireConformanceToRequestAnthropicToolWire); - failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector135WireConformanceToRequestChatAudioMp3); - failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector136WireConformanceToRequestChatAudioPart); - failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector137WireConformanceToRequestChatImageBase64); - failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector138WireConformanceToRequestChatImagePart); - failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector139WireConformanceToRequestChatMultipartContent); - failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector140WireConformanceToRequestChatSimple); - failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector141WireConformanceToRequestChatSingleTextOptimized); - failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector142WireConformanceToRequestChatWithOptions); - failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector143WireConformanceToRequestEmbeddingWire); - failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector144WireConformanceToRequestImageWire); - failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector145WireConformanceToRequestKindToJsonTypeMapping); - failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector146WireConformanceToRequestOptionsAdditionalProperties); - failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector147WireConformanceToRequestOptionsMaxCompletionTokens); - failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector148WireConformanceToRequestOptionsStopSequences); - failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector149WireConformanceToRequestResponsesSimple); - failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector150WireConformanceToRequestResponsesStructuredOutput); - failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector151WireConformanceToRequestResponsesWithTools); - failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector152WireConformanceToRequestStructuredOutput); - failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector153WireConformanceToRequestStructuredOutputNestedOptional); - failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector154WireConformanceToRequestToolsBindingsStripped); - failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector155WireConformanceToRequestToolsFunctionWire); - failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector156WireConformanceToRequestToolsNullWhenEmpty); - failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector157WireConformanceToRequestToolsStrictMode); + failed += runCase("LoadConformance.load:untrusted_endpoint_blank_literal_key_rejected", VectorConformanceTests::vector58LoadConformanceLoadUntrustedEndpointBlankLiteralKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_env_default_rejected", VectorConformanceTests::vector59LoadConformanceLoadUntrustedEndpointEnvDefaultRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_env_key_rejected", VectorConformanceTests::vector60LoadConformanceLoadUntrustedEndpointEnvKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_file_connection_rejected", VectorConformanceTests::vector61LoadConformanceLoadUntrustedEndpointFileConnectionRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_fragment_before_at_rejected", VectorConformanceTests::vector62LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_ipv6_literal_rejected", VectorConformanceTests::vector63LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", VectorConformanceTests::vector64LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", VectorConformanceTests::vector65LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_no_key_rejected", VectorConformanceTests::vector66LoadConformanceLoadUntrustedEndpointNoKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_oauth_literal_secret_rejected", VectorConformanceTests::vector67LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_percent_encoded_host_rejected", VectorConformanceTests::vector68LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_port_mismatch_rejected", VectorConformanceTests::vector69LoadConformanceLoadUntrustedEndpointPortMismatchRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_query_before_at_rejected", VectorConformanceTests::vector70LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_scheme_mismatch_rejected", VectorConformanceTests::vector71LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_lookalike_rejected", VectorConformanceTests::vector72LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", VectorConformanceTests::vector73LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_trailing_dot_rejected", VectorConformanceTests::vector74LoadConformanceLoadUntrustedEndpointTrailingDotRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_unicode_homoglyph_rejected", VectorConformanceTests::vector75LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_userinfo_lookalike_rejected", VectorConformanceTests::vector76LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", VectorConformanceTests::vector77LoadConformanceLoadUntrustedEndpointWildcardApexRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_boundary_rejected", VectorConformanceTests::vector78LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected); + failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector79MemoryConformanceOperateClearByCategoryAndAll); + failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector80MemoryConformanceOperateFormatCoreAndRecallResults); + failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector81MemoryConformanceOperateRecallEmptyQueryAndLimit); + failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector82MemoryConformanceOperateRecallRanksByWeightedScore); + failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector83MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); + failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector84MemoryConformanceOperateRecallTagsAndCoreBoost); + failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector85MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); + failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector86MemoryConformanceOperateRememberDedupsCoreByTags); + failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector87MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); + failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector88MemoryConformanceOperateRememberEvictsArchivalFirst); + failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector89MemoryConformanceOperateSnapshotPortRoundTrip); + failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector90MemoryConformanceOperateUpdateIndexOutOfRangeError); + failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector91TurnConformanceReplayMaxIterations); + failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector92TurnConformanceReplayNoTool); + failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector93TurnConformanceReplayPermissionDenied); + failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector94TurnConformanceReplayToolFailure); + failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector95TurnConformanceReplayToolSuccess); + failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector96TurnConformanceRunAssistantToolCallsMetadata); + failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector97TurnConformanceRunAsyncToolFunction); + failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector98TurnConformanceRunBindingsInjected); + failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector99TurnConformanceRunCancellationBeforeLlm); + failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector100TurnConformanceRunCancellationBetweenIterations); + failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector101TurnConformanceRunCancellationBetweenTools); + failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector102TurnConformanceRunContextNoTrimWhenFits); + failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector103TurnConformanceRunContextPreservesSystemMessages); + failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector104TurnConformanceRunContextTrimBasic); + failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector105TurnConformanceRunEmptyToolResult); + failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector106TurnConformanceRunEventsBasicToolLoop); + failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector107TurnConformanceRunEventsErrorLogged); + failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector108TurnConformanceRunEventsNoTools); + failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector109TurnConformanceRunGuardrailAllPass); + failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector110TurnConformanceRunGuardrailInputDeny); + failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector111TurnConformanceRunGuardrailOutputDeny); + failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector112TurnConformanceRunGuardrailToolDeny); + failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector113TurnConformanceRunMaxIterationsExceeded); + failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector114TurnConformanceRunMultiTurnToolCalls); + failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector115TurnConformanceRunMultipleToolCallsSingleTurn); + failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector116TurnConformanceRunNoToolCalls); + failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector117TurnConformanceRunParallelToolsBasic); + failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector118TurnConformanceRunParallelToolsWithGuardrailDeny); + failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector119TurnConformanceRunParallelToolsWithToolError); + failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector120TurnConformanceRunSingleToolCall); + failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector121TurnConformanceRunSteeringInjectMessage); + failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector122TurnConformanceRunSteeringMultipleMessages); + failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector123TurnConformanceRunToolNotRegisteredError); + failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector124TurnConformanceRunToolResultMessageFormat); + failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector125TurnConformanceRunTurnCancelBeforeContext); + failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector126TurnConformanceRunTurnDelegatedProviderState); + failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector127TurnConformanceRunTurnFinalOutput); + failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector128TurnConformanceRunTurnMemoryContextIsModelVisible); + failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector129TurnConformanceRunTurnOrderedToolRound); + failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector130TurnConformanceRunTurnPermissionDenialIsModelVisible); + failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector131WireConformanceToRequestAnthropicImageFormat); + failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector132WireConformanceToRequestAnthropicMaxTokensRequired); + failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector133WireConformanceToRequestAnthropicOptions); + failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector134WireConformanceToRequestAnthropicSystemSeparate); + failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector135WireConformanceToRequestAnthropicToolWire); + failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector136WireConformanceToRequestChatAudioMp3); + failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector137WireConformanceToRequestChatAudioPart); + failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector138WireConformanceToRequestChatImageBase64); + failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector139WireConformanceToRequestChatImagePart); + failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector140WireConformanceToRequestChatMultipartContent); + failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector141WireConformanceToRequestChatSimple); + failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector142WireConformanceToRequestChatSingleTextOptimized); + failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector143WireConformanceToRequestChatWithOptions); + failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector144WireConformanceToRequestEmbeddingWire); + failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector145WireConformanceToRequestImageWire); + failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector146WireConformanceToRequestKindToJsonTypeMapping); + failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector147WireConformanceToRequestOptionsAdditionalProperties); + failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector148WireConformanceToRequestOptionsMaxCompletionTokens); + failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector149WireConformanceToRequestOptionsStopSequences); + failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector150WireConformanceToRequestResponsesSimple); + failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector151WireConformanceToRequestResponsesStructuredOutput); + failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector152WireConformanceToRequestResponsesWithTools); + failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector153WireConformanceToRequestStructuredOutput); + failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector154WireConformanceToRequestStructuredOutputNestedOptional); + failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector155WireConformanceToRequestToolsBindingsStripped); + failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector156WireConformanceToRequestToolsFunctionWire); + failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector157WireConformanceToRequestToolsNullWhenEmpty); + failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector158WireConformanceToRequestToolsStrictMode); if (failed > 0) { throw new AssertionError(failed + " @vector conformance failure(s)"); } diff --git a/runtime/python/prompty/prompty/core/endpoint_trust.py b/runtime/python/prompty/prompty/core/endpoint_trust.py index d57549ed8..55f962575 100644 --- a/runtime/python/prompty/prompty/core/endpoint_trust.py +++ b/runtime/python/prompty/prompty/core/endpoint_trust.py @@ -89,7 +89,8 @@ def _file_supplied_credential(connection: Mapping[str, Any]) -> bool: if key is None: return False value = connection.get(key) - return isinstance(value, str) and bool(value) and not _is_reference(value) + # Blank keys count as absent: executors that trim fall back to ambient credentials. + return isinstance(value, str) and bool(value.strip()) and not _is_reference(value) def _file_controlled_destination(value: Any) -> str | None: diff --git a/runtime/python/prompty/prompty/model/vector_conformance.py b/runtime/python/prompty/prompty/model/vector_conformance.py index f0505d823..0c1f7975b 100644 --- a/runtime/python/prompty/prompty/model/vector_conformance.py +++ b/runtime/python/prompty/prompty/model/vector_conformance.py @@ -555,6 +555,16 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: except Exception as error: # noqa: BLE001 caught = error assert caught is not None, "untrusted_endpoint_backslash_rejected: expected an error" + # vector: untrusted_endpoint_blank_literal_key_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_blank_literal_key_rejected: expected an error" # vector: untrusted_endpoint_env_default_rejected source = json.loads("null") caught = None diff --git a/runtime/python/prompty/tests/model/test_vector_conformance.py b/runtime/python/prompty/tests/model/test_vector_conformance.py index 0f41aebdc..e2184ff02 100644 --- a/runtime/python/prompty/tests/model/test_vector_conformance.py +++ b/runtime/python/prompty/tests/model/test_vector_conformance.py @@ -2032,7 +2032,37 @@ async def test_vector_57_loadconformance_load_untrusted_endpoint_backslash_rejec await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_58_loadconformance_load_untrusted_endpoint_env_default_rejected(): +async def test_vector_58_loadconformance_load_untrusted_endpoint_blank_literal_key_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_blank_literal_key_rejected", + "description": "a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-blank-key", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": " " + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_59_loadconformance_load_untrusted_endpoint_env_default_rejected(): vector_json = r""" { "name": "untrusted_endpoint_env_default_rejected", @@ -2061,7 +2091,7 @@ async def test_vector_58_loadconformance_load_untrusted_endpoint_env_default_rej await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_59_loadconformance_load_untrusted_endpoint_env_key_rejected(): +async def test_vector_60_loadconformance_load_untrusted_endpoint_env_key_rejected(): vector_json = r""" { "name": "untrusted_endpoint_env_key_rejected", @@ -2093,7 +2123,7 @@ async def test_vector_59_loadconformance_load_untrusted_endpoint_env_key_rejecte await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_60_loadconformance_load_untrusted_endpoint_file_connection_rejected(): +async def test_vector_61_loadconformance_load_untrusted_endpoint_file_connection_rejected(): vector_json = r""" { "name": "untrusted_endpoint_file_connection_rejected", @@ -2124,7 +2154,7 @@ async def test_vector_60_loadconformance_load_untrusted_endpoint_file_connection await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_61_loadconformance_load_untrusted_endpoint_fragment_before_at_rejected(): +async def test_vector_62_loadconformance_load_untrusted_endpoint_fragment_before_at_rejected(): vector_json = r""" { "name": "untrusted_endpoint_fragment_before_at_rejected", @@ -2152,7 +2182,7 @@ async def test_vector_61_loadconformance_load_untrusted_endpoint_fragment_before await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_62_loadconformance_load_untrusted_endpoint_ipv6_literal_rejected(): +async def test_vector_63_loadconformance_load_untrusted_endpoint_ipv6_literal_rejected(): vector_json = r""" { "name": "untrusted_endpoint_ipv6_literal_rejected", @@ -2180,7 +2210,7 @@ async def test_vector_62_loadconformance_load_untrusted_endpoint_ipv6_literal_re await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_63_loadconformance_load_untrusted_endpoint_key_with_client_secret_rejected(): +async def test_vector_64_loadconformance_load_untrusted_endpoint_key_with_client_secret_rejected(): vector_json = r""" { "name": "untrusted_endpoint_key_with_client_secret_rejected", @@ -2209,7 +2239,7 @@ async def test_vector_63_loadconformance_load_untrusted_endpoint_key_with_client await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_64_loadconformance_load_untrusted_endpoint_mismatched_credential_rejected(): +async def test_vector_65_loadconformance_load_untrusted_endpoint_mismatched_credential_rejected(): vector_json = r""" { "name": "untrusted_endpoint_mismatched_credential_rejected", @@ -2238,7 +2268,7 @@ async def test_vector_64_loadconformance_load_untrusted_endpoint_mismatched_cred await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_65_loadconformance_load_untrusted_endpoint_no_key_rejected(): +async def test_vector_66_loadconformance_load_untrusted_endpoint_no_key_rejected(): vector_json = r""" { "name": "untrusted_endpoint_no_key_rejected", @@ -2266,7 +2296,7 @@ async def test_vector_65_loadconformance_load_untrusted_endpoint_no_key_rejected await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_66_loadconformance_load_untrusted_endpoint_oauth_literal_secret_rejected(): +async def test_vector_67_loadconformance_load_untrusted_endpoint_oauth_literal_secret_rejected(): vector_json = r""" { "name": "untrusted_endpoint_oauth_literal_secret_rejected", @@ -2298,7 +2328,7 @@ async def test_vector_66_loadconformance_load_untrusted_endpoint_oauth_literal_s await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_67_loadconformance_load_untrusted_endpoint_percent_encoded_host_rejected(): +async def test_vector_68_loadconformance_load_untrusted_endpoint_percent_encoded_host_rejected(): vector_json = r""" { "name": "untrusted_endpoint_percent_encoded_host_rejected", @@ -2326,7 +2356,7 @@ async def test_vector_67_loadconformance_load_untrusted_endpoint_percent_encoded await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_68_loadconformance_load_untrusted_endpoint_port_mismatch_rejected(): +async def test_vector_69_loadconformance_load_untrusted_endpoint_port_mismatch_rejected(): vector_json = r""" { "name": "untrusted_endpoint_port_mismatch_rejected", @@ -2354,7 +2384,7 @@ async def test_vector_68_loadconformance_load_untrusted_endpoint_port_mismatch_r await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_69_loadconformance_load_untrusted_endpoint_query_before_at_rejected(): +async def test_vector_70_loadconformance_load_untrusted_endpoint_query_before_at_rejected(): vector_json = r""" { "name": "untrusted_endpoint_query_before_at_rejected", @@ -2382,7 +2412,7 @@ async def test_vector_69_loadconformance_load_untrusted_endpoint_query_before_at await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_70_loadconformance_load_untrusted_endpoint_scheme_mismatch_rejected(): +async def test_vector_71_loadconformance_load_untrusted_endpoint_scheme_mismatch_rejected(): vector_json = r""" { "name": "untrusted_endpoint_scheme_mismatch_rejected", @@ -2410,7 +2440,7 @@ async def test_vector_70_loadconformance_load_untrusted_endpoint_scheme_mismatch await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_71_loadconformance_load_untrusted_endpoint_suffix_lookalike_rejected(): +async def test_vector_72_loadconformance_load_untrusted_endpoint_suffix_lookalike_rejected(): vector_json = r""" { "name": "untrusted_endpoint_suffix_lookalike_rejected", @@ -2438,7 +2468,7 @@ async def test_vector_71_loadconformance_load_untrusted_endpoint_suffix_lookalik await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_72_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected(): +async def test_vector_73_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected(): vector_json = r""" { "name": "untrusted_endpoint_suffix_spoof_rejected", @@ -2469,7 +2499,7 @@ async def test_vector_72_loadconformance_load_untrusted_endpoint_suffix_spoof_re await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_73_loadconformance_load_untrusted_endpoint_trailing_dot_rejected(): +async def test_vector_74_loadconformance_load_untrusted_endpoint_trailing_dot_rejected(): vector_json = r""" { "name": "untrusted_endpoint_trailing_dot_rejected", @@ -2497,7 +2527,7 @@ async def test_vector_73_loadconformance_load_untrusted_endpoint_trailing_dot_re await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_74_loadconformance_load_untrusted_endpoint_unicode_homoglyph_rejected(): +async def test_vector_75_loadconformance_load_untrusted_endpoint_unicode_homoglyph_rejected(): vector_json = r""" { "name": "untrusted_endpoint_unicode_homoglyph_rejected", @@ -2525,7 +2555,7 @@ async def test_vector_74_loadconformance_load_untrusted_endpoint_unicode_homogly await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_75_loadconformance_load_untrusted_endpoint_userinfo_lookalike_rejected(): +async def test_vector_76_loadconformance_load_untrusted_endpoint_userinfo_lookalike_rejected(): vector_json = r""" { "name": "untrusted_endpoint_userinfo_lookalike_rejected", @@ -2553,7 +2583,7 @@ async def test_vector_75_loadconformance_load_untrusted_endpoint_userinfo_lookal await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_76_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected(): +async def test_vector_77_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected(): vector_json = r""" { "name": "untrusted_endpoint_wildcard_apex_rejected", @@ -2581,7 +2611,7 @@ async def test_vector_76_loadconformance_load_untrusted_endpoint_wildcard_apex_r await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_77_loadconformance_load_untrusted_endpoint_wildcard_boundary_rejected(): +async def test_vector_78_loadconformance_load_untrusted_endpoint_wildcard_boundary_rejected(): vector_json = r""" { "name": "untrusted_endpoint_wildcard_boundary_rejected", @@ -2609,7 +2639,7 @@ async def test_vector_77_loadconformance_load_untrusted_endpoint_wildcard_bounda await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_78_memoryconformance_operate_clear_by_category_and_all(): +async def test_vector_79_memoryconformance_operate_clear_by_category_and_all(): vector_json = r""" { "name": "clear_by_category_and_all", @@ -2661,7 +2691,7 @@ async def test_vector_78_memoryconformance_operate_clear_by_category_and_all(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_79_memoryconformance_operate_format_core_and_recall_results(): +async def test_vector_80_memoryconformance_operate_format_core_and_recall_results(): vector_json = r""" { "name": "format_core_and_recall_results", @@ -2701,7 +2731,7 @@ async def test_vector_79_memoryconformance_operate_format_core_and_recall_result await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_80_memoryconformance_operate_recall_empty_query_and_limit(): +async def test_vector_81_memoryconformance_operate_recall_empty_query_and_limit(): vector_json = r""" { "name": "recall_empty_query_and_limit", @@ -2754,7 +2784,7 @@ async def test_vector_80_memoryconformance_operate_recall_empty_query_and_limit( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_81_memoryconformance_operate_recall_ranks_by_weighted_score(): +async def test_vector_82_memoryconformance_operate_recall_ranks_by_weighted_score(): vector_json = r""" { "name": "recall_ranks_by_weighted_score", @@ -2811,7 +2841,7 @@ async def test_vector_81_memoryconformance_operate_recall_ranks_by_weighted_scor await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_82_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): +async def test_vector_83_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): vector_json = r""" { "name": "recall_score_ties_preserve_insertion_order", @@ -2867,7 +2897,7 @@ async def test_vector_82_memoryconformance_operate_recall_score_ties_preserve_in await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_83_memoryconformance_operate_recall_tags_and_core_boost(): +async def test_vector_84_memoryconformance_operate_recall_tags_and_core_boost(): vector_json = r""" { "name": "recall_tags_and_core_boost", @@ -2926,7 +2956,7 @@ async def test_vector_83_memoryconformance_operate_recall_tags_and_core_boost(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_84_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): +async def test_vector_85_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): vector_json = r""" { "name": "recall_tokenization_punctuation_unicode_and_dedup", @@ -2985,7 +3015,7 @@ async def test_vector_84_memoryconformance_operate_recall_tokenization_punctuati await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_85_memoryconformance_operate_remember_dedups_core_by_tags(): +async def test_vector_86_memoryconformance_operate_remember_dedups_core_by_tags(): vector_json = r""" { "name": "remember_dedups_core_by_tags", @@ -3047,7 +3077,7 @@ async def test_vector_85_memoryconformance_operate_remember_dedups_core_by_tags( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_86_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): +async def test_vector_87_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): vector_json = r""" { "name": "remember_dedups_core_missing_and_empty_tags", @@ -3101,7 +3131,7 @@ async def test_vector_86_memoryconformance_operate_remember_dedups_core_missing_ await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_87_memoryconformance_operate_remember_evicts_archival_first(): +async def test_vector_88_memoryconformance_operate_remember_evicts_archival_first(): vector_json = r""" { "name": "remember_evicts_archival_first", @@ -3160,7 +3190,7 @@ async def test_vector_87_memoryconformance_operate_remember_evicts_archival_firs await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_88_memoryconformance_operate_snapshot_port_round_trip(): +async def test_vector_89_memoryconformance_operate_snapshot_port_round_trip(): vector_json = r""" { "name": "snapshot_port_round_trip", @@ -3198,7 +3228,7 @@ async def test_vector_88_memoryconformance_operate_snapshot_port_round_trip(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_89_memoryconformance_operate_update_index_out_of_range_error(): +async def test_vector_90_memoryconformance_operate_update_index_out_of_range_error(): vector_json = r""" { "name": "update_index_out_of_range_error", @@ -3230,7 +3260,7 @@ async def test_vector_89_memoryconformance_operate_update_index_out_of_range_err await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_90_turnconformance_replay_max_iterations(): +async def test_vector_91_turnconformance_replay_max_iterations(): vector_json = r""" { "name": "max_iterations", @@ -3265,7 +3295,7 @@ async def test_vector_90_turnconformance_replay_max_iterations(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_91_turnconformance_replay_no_tool(): +async def test_vector_92_turnconformance_replay_no_tool(): vector_json = r""" { "name": "no_tool", @@ -3296,7 +3326,7 @@ async def test_vector_91_turnconformance_replay_no_tool(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_92_turnconformance_replay_permission_denied(): +async def test_vector_93_turnconformance_replay_permission_denied(): vector_json = r""" { "name": "permission_denied", @@ -3330,7 +3360,7 @@ async def test_vector_92_turnconformance_replay_permission_denied(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_93_turnconformance_replay_tool_failure(): +async def test_vector_94_turnconformance_replay_tool_failure(): vector_json = r""" { "name": "tool_failure", @@ -3366,7 +3396,7 @@ async def test_vector_93_turnconformance_replay_tool_failure(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_94_turnconformance_replay_tool_success(): +async def test_vector_95_turnconformance_replay_tool_success(): vector_json = r""" { "name": "tool_success", @@ -3402,7 +3432,7 @@ async def test_vector_94_turnconformance_replay_tool_success(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_95_turnconformance_run_assistant_tool_calls_metadata(): +async def test_vector_96_turnconformance_run_assistant_tool_calls_metadata(): vector_json = r""" { "name": "assistant_tool_calls_metadata", @@ -3554,7 +3584,7 @@ async def test_vector_95_turnconformance_run_assistant_tool_calls_metadata(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_96_turnconformance_run_async_tool_function(): +async def test_vector_97_turnconformance_run_async_tool_function(): vector_json = r""" { "name": "async_tool_function", @@ -3664,7 +3694,7 @@ async def test_vector_96_turnconformance_run_async_tool_function(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_97_turnconformance_run_bindings_injected(): +async def test_vector_98_turnconformance_run_bindings_injected(): vector_json = r""" { "name": "bindings_injected", @@ -3794,7 +3824,7 @@ async def test_vector_97_turnconformance_run_bindings_injected(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_98_turnconformance_run_cancellation_before_llm(): +async def test_vector_99_turnconformance_run_cancellation_before_llm(): vector_json = r""" { "name": "cancellation_before_llm", @@ -3855,7 +3885,7 @@ async def test_vector_98_turnconformance_run_cancellation_before_llm(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_99_turnconformance_run_cancellation_between_iterations(): +async def test_vector_100_turnconformance_run_cancellation_between_iterations(): vector_json = r""" { "name": "cancellation_between_iterations", @@ -3985,7 +4015,7 @@ async def test_vector_99_turnconformance_run_cancellation_between_iterations(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_100_turnconformance_run_cancellation_between_tools(): +async def test_vector_101_turnconformance_run_cancellation_between_tools(): vector_json = r""" { "name": "cancellation_between_tools", @@ -4124,7 +4154,7 @@ async def test_vector_100_turnconformance_run_cancellation_between_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_101_turnconformance_run_context_no_trim_when_fits(): +async def test_vector_102_turnconformance_run_context_no_trim_when_fits(): vector_json = r""" { "name": "context_no_trim_when_fits", @@ -4195,7 +4225,7 @@ async def test_vector_101_turnconformance_run_context_no_trim_when_fits(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_102_turnconformance_run_context_preserves_system_messages(): +async def test_vector_103_turnconformance_run_context_preserves_system_messages(): vector_json = r""" { "name": "context_preserves_system_messages", @@ -4302,7 +4332,7 @@ async def test_vector_102_turnconformance_run_context_preserves_system_messages( await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_103_turnconformance_run_context_trim_basic(): +async def test_vector_104_turnconformance_run_context_trim_basic(): vector_json = r""" { "name": "context_trim_basic", @@ -4460,7 +4490,7 @@ async def test_vector_103_turnconformance_run_context_trim_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_104_turnconformance_run_empty_tool_result(): +async def test_vector_105_turnconformance_run_empty_tool_result(): vector_json = r""" { "name": "empty_tool_result", @@ -4575,7 +4605,7 @@ async def test_vector_104_turnconformance_run_empty_tool_result(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_105_turnconformance_run_events_basic_tool_loop(): +async def test_vector_106_turnconformance_run_events_basic_tool_loop(): vector_json = r""" { "name": "events_basic_tool_loop", @@ -4721,7 +4751,7 @@ async def test_vector_105_turnconformance_run_events_basic_tool_loop(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_106_turnconformance_run_events_error_logged(): +async def test_vector_107_turnconformance_run_events_error_logged(): vector_json = r""" { "name": "events_error_logged", @@ -4847,7 +4877,7 @@ async def test_vector_106_turnconformance_run_events_error_logged(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_107_turnconformance_run_events_no_tools(): +async def test_vector_108_turnconformance_run_events_no_tools(): vector_json = r""" { "name": "events_no_tools", @@ -4925,7 +4955,7 @@ async def test_vector_107_turnconformance_run_events_no_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_108_turnconformance_run_guardrail_all_pass(): +async def test_vector_109_turnconformance_run_guardrail_all_pass(): vector_json = r""" { "name": "guardrail_all_pass", @@ -5048,7 +5078,7 @@ async def test_vector_108_turnconformance_run_guardrail_all_pass(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_109_turnconformance_run_guardrail_input_deny(): +async def test_vector_110_turnconformance_run_guardrail_input_deny(): vector_json = r""" { "name": "guardrail_input_deny", @@ -5104,7 +5134,7 @@ async def test_vector_109_turnconformance_run_guardrail_input_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_110_turnconformance_run_guardrail_output_deny(): +async def test_vector_111_turnconformance_run_guardrail_output_deny(): vector_json = r""" { "name": "guardrail_output_deny", @@ -5182,7 +5212,7 @@ async def test_vector_110_turnconformance_run_guardrail_output_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_111_turnconformance_run_guardrail_tool_deny(): +async def test_vector_112_turnconformance_run_guardrail_tool_deny(): vector_json = r""" { "name": "guardrail_tool_deny", @@ -5336,7 +5366,7 @@ async def test_vector_111_turnconformance_run_guardrail_tool_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_112_turnconformance_run_max_iterations_exceeded(): +async def test_vector_113_turnconformance_run_max_iterations_exceeded(): vector_json = r""" { "name": "max_iterations_exceeded", @@ -5846,7 +5876,7 @@ async def test_vector_112_turnconformance_run_max_iterations_exceeded(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_113_turnconformance_run_multi_turn_tool_calls(): +async def test_vector_114_turnconformance_run_multi_turn_tool_calls(): vector_json = r""" { "name": "multi_turn_tool_calls", @@ -6023,7 +6053,7 @@ async def test_vector_113_turnconformance_run_multi_turn_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_114_turnconformance_run_multiple_tool_calls_single_turn(): +async def test_vector_115_turnconformance_run_multiple_tool_calls_single_turn(): vector_json = r""" { "name": "multiple_tool_calls_single_turn", @@ -6204,7 +6234,7 @@ async def test_vector_114_turnconformance_run_multiple_tool_calls_single_turn(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_115_turnconformance_run_no_tool_calls(): +async def test_vector_116_turnconformance_run_no_tool_calls(): vector_json = r""" { "name": "no_tool_calls", @@ -6272,7 +6302,7 @@ async def test_vector_115_turnconformance_run_no_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_116_turnconformance_run_parallel_tools_basic(): +async def test_vector_117_turnconformance_run_parallel_tools_basic(): vector_json = r""" { "name": "parallel_tools_basic", @@ -6445,7 +6475,7 @@ async def test_vector_116_turnconformance_run_parallel_tools_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_117_turnconformance_run_parallel_tools_with_guardrail_deny(): +async def test_vector_118_turnconformance_run_parallel_tools_with_guardrail_deny(): vector_json = r""" { "name": "parallel_tools_with_guardrail_deny", @@ -6636,7 +6666,7 @@ async def test_vector_117_turnconformance_run_parallel_tools_with_guardrail_deny await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_118_turnconformance_run_parallel_tools_with_tool_error(): +async def test_vector_119_turnconformance_run_parallel_tools_with_tool_error(): vector_json = r""" { "name": "parallel_tools_with_tool_error", @@ -6809,7 +6839,7 @@ async def test_vector_118_turnconformance_run_parallel_tools_with_tool_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_119_turnconformance_run_single_tool_call(): +async def test_vector_120_turnconformance_run_single_tool_call(): vector_json = r""" { "name": "single_tool_call", @@ -6956,7 +6986,7 @@ async def test_vector_119_turnconformance_run_single_tool_call(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_120_turnconformance_run_steering_inject_message(): +async def test_vector_121_turnconformance_run_steering_inject_message(): vector_json = r""" { "name": "steering_inject_message", @@ -7164,7 +7194,7 @@ async def test_vector_120_turnconformance_run_steering_inject_message(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_121_turnconformance_run_steering_multiple_messages(): +async def test_vector_122_turnconformance_run_steering_multiple_messages(): vector_json = r""" { "name": "steering_multiple_messages", @@ -7334,7 +7364,7 @@ async def test_vector_121_turnconformance_run_steering_multiple_messages(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_122_turnconformance_run_tool_not_registered_error(): +async def test_vector_123_turnconformance_run_tool_not_registered_error(): vector_json = r""" { "name": "tool_not_registered_error", @@ -7418,7 +7448,7 @@ async def test_vector_122_turnconformance_run_tool_not_registered_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_123_turnconformance_run_tool_result_message_format(): +async def test_vector_124_turnconformance_run_tool_result_message_format(): vector_json = r""" { "name": "tool_result_message_format", @@ -7539,7 +7569,7 @@ async def test_vector_123_turnconformance_run_tool_result_message_format(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_124_turnconformance_runturn_cancel_before_context(): +async def test_vector_125_turnconformance_runturn_cancel_before_context(): vector_json = r""" { "name": "cancel_before_context", @@ -7571,7 +7601,7 @@ async def test_vector_124_turnconformance_runturn_cancel_before_context(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_125_turnconformance_runturn_delegated_provider_state(): +async def test_vector_126_turnconformance_runturn_delegated_provider_state(): vector_json = r""" { "name": "delegated_provider_state", @@ -7638,7 +7668,7 @@ async def test_vector_125_turnconformance_runturn_delegated_provider_state(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_126_turnconformance_runturn_final_output(): +async def test_vector_127_turnconformance_runturn_final_output(): vector_json = r""" { "name": "final_output", @@ -7683,7 +7713,7 @@ async def test_vector_126_turnconformance_runturn_final_output(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_127_turnconformance_runturn_memory_context_is_model_visible(): +async def test_vector_128_turnconformance_runturn_memory_context_is_model_visible(): vector_json = r""" { "name": "memory_context_is_model_visible", @@ -7745,7 +7775,7 @@ async def test_vector_127_turnconformance_runturn_memory_context_is_model_visibl await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_128_turnconformance_runturn_ordered_tool_round(): +async def test_vector_129_turnconformance_runturn_ordered_tool_round(): vector_json = r""" { "name": "ordered_tool_round", @@ -7836,7 +7866,7 @@ async def test_vector_128_turnconformance_runturn_ordered_tool_round(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_129_turnconformance_runturn_permission_denial_is_model_visible(): +async def test_vector_130_turnconformance_runturn_permission_denial_is_model_visible(): vector_json = r""" { "name": "permission_denial_is_model_visible", @@ -7887,7 +7917,7 @@ async def test_vector_129_turnconformance_runturn_permission_denial_is_model_vis await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_130_wireconformance_torequest_anthropic_image_format(): +async def test_vector_131_wireconformance_torequest_anthropic_image_format(): vector_json = r""" { "name": "anthropic_image_format", @@ -7957,7 +7987,7 @@ async def test_vector_130_wireconformance_torequest_anthropic_image_format(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_131_wireconformance_torequest_anthropic_max_tokens_required(): +async def test_vector_132_wireconformance_torequest_anthropic_max_tokens_required(): vector_json = r""" { "name": "anthropic_max_tokens_required", @@ -8012,7 +8042,7 @@ async def test_vector_131_wireconformance_torequest_anthropic_max_tokens_require await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_132_wireconformance_torequest_anthropic_options(): +async def test_vector_133_wireconformance_torequest_anthropic_options(): vector_json = r""" { "name": "anthropic_options", @@ -8081,7 +8111,7 @@ async def test_vector_132_wireconformance_torequest_anthropic_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_133_wireconformance_torequest_anthropic_system_separate(): +async def test_vector_134_wireconformance_torequest_anthropic_system_separate(): vector_json = r""" { "name": "anthropic_system_separate", @@ -8148,7 +8178,7 @@ async def test_vector_133_wireconformance_torequest_anthropic_system_separate(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_134_wireconformance_torequest_anthropic_tool_wire(): +async def test_vector_135_wireconformance_torequest_anthropic_tool_wire(): vector_json = r""" { "name": "anthropic_tool_wire", @@ -8233,7 +8263,7 @@ async def test_vector_134_wireconformance_torequest_anthropic_tool_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_135_wireconformance_torequest_chat_audio_mp3(): +async def test_vector_136_wireconformance_torequest_chat_audio_mp3(): vector_json = r""" { "name": "chat_audio_mp3", @@ -8291,7 +8321,7 @@ async def test_vector_135_wireconformance_torequest_chat_audio_mp3(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_136_wireconformance_torequest_chat_audio_part(): +async def test_vector_137_wireconformance_torequest_chat_audio_part(): vector_json = r""" { "name": "chat_audio_part", @@ -8349,7 +8379,7 @@ async def test_vector_136_wireconformance_torequest_chat_audio_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_137_wireconformance_torequest_chat_image_base64(): +async def test_vector_138_wireconformance_torequest_chat_image_base64(): vector_json = r""" { "name": "chat_image_base64", @@ -8413,7 +8443,7 @@ async def test_vector_137_wireconformance_torequest_chat_image_base64(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_138_wireconformance_torequest_chat_image_part(): +async def test_vector_139_wireconformance_torequest_chat_image_part(): vector_json = r""" { "name": "chat_image_part", @@ -8477,7 +8507,7 @@ async def test_vector_138_wireconformance_torequest_chat_image_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_139_wireconformance_torequest_chat_multipart_content(): +async def test_vector_140_wireconformance_torequest_chat_multipart_content(): vector_json = r""" { "name": "chat_multipart_content", @@ -8541,7 +8571,7 @@ async def test_vector_139_wireconformance_torequest_chat_multipart_content(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_140_wireconformance_torequest_chat_simple(): +async def test_vector_141_wireconformance_torequest_chat_simple(): vector_json = r""" { "name": "chat_simple", @@ -8603,7 +8633,7 @@ async def test_vector_140_wireconformance_torequest_chat_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_141_wireconformance_torequest_chat_single_text_optimized(): +async def test_vector_142_wireconformance_torequest_chat_single_text_optimized(): vector_json = r""" { "name": "chat_single_text_optimized", @@ -8652,7 +8682,7 @@ async def test_vector_141_wireconformance_torequest_chat_single_text_optimized() await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_142_wireconformance_torequest_chat_with_options(): +async def test_vector_143_wireconformance_torequest_chat_with_options(): vector_json = r""" { "name": "chat_with_options", @@ -8720,7 +8750,7 @@ async def test_vector_142_wireconformance_torequest_chat_with_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_143_wireconformance_torequest_embedding_wire(): +async def test_vector_144_wireconformance_torequest_embedding_wire(): vector_json = r""" { "name": "embedding_wire", @@ -8764,7 +8794,7 @@ async def test_vector_143_wireconformance_torequest_embedding_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_144_wireconformance_torequest_image_wire(): +async def test_vector_145_wireconformance_torequest_image_wire(): vector_json = r""" { "name": "image_wire", @@ -8808,7 +8838,7 @@ async def test_vector_144_wireconformance_torequest_image_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_145_wireconformance_torequest_kind_to_json_type_mapping(): +async def test_vector_146_wireconformance_torequest_kind_to_json_type_mapping(): vector_json = r""" { "name": "kind_to_json_type_mapping", @@ -8927,7 +8957,7 @@ async def test_vector_145_wireconformance_torequest_kind_to_json_type_mapping(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_146_wireconformance_torequest_options_additional_properties(): +async def test_vector_147_wireconformance_torequest_options_additional_properties(): vector_json = r""" { "name": "options_additional_properties", @@ -8981,7 +9011,7 @@ async def test_vector_146_wireconformance_torequest_options_additional_propertie await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_147_wireconformance_torequest_options_max_completion_tokens(): +async def test_vector_148_wireconformance_torequest_options_max_completion_tokens(): vector_json = r""" { "name": "options_max_completion_tokens", @@ -9033,7 +9063,7 @@ async def test_vector_147_wireconformance_torequest_options_max_completion_token await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_148_wireconformance_torequest_options_stop_sequences(): +async def test_vector_149_wireconformance_torequest_options_stop_sequences(): vector_json = r""" { "name": "options_stop_sequences", @@ -9091,7 +9121,7 @@ async def test_vector_148_wireconformance_torequest_options_stop_sequences(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_149_wireconformance_torequest_responses_simple(): +async def test_vector_150_wireconformance_torequest_responses_simple(): vector_json = r""" { "name": "responses_simple", @@ -9150,7 +9180,7 @@ async def test_vector_149_wireconformance_torequest_responses_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_150_wireconformance_torequest_responses_structured_output(): +async def test_vector_151_wireconformance_torequest_responses_structured_output(): vector_json = r""" { "name": "responses_structured_output", @@ -9237,7 +9267,7 @@ async def test_vector_150_wireconformance_torequest_responses_structured_output( await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_151_wireconformance_torequest_responses_with_tools(): +async def test_vector_152_wireconformance_torequest_responses_with_tools(): vector_json = r""" { "name": "responses_with_tools", @@ -9317,7 +9347,7 @@ async def test_vector_151_wireconformance_torequest_responses_with_tools(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_152_wireconformance_torequest_structured_output(): +async def test_vector_153_wireconformance_torequest_structured_output(): vector_json = r""" { "name": "structured_output", @@ -9400,7 +9430,7 @@ async def test_vector_152_wireconformance_torequest_structured_output(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_153_wireconformance_torequest_structured_output_nested_optional(): +async def test_vector_154_wireconformance_torequest_structured_output_nested_optional(): vector_json = r""" { "name": "structured_output_nested_optional", @@ -9502,7 +9532,7 @@ async def test_vector_153_wireconformance_torequest_structured_output_nested_opt await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_154_wireconformance_torequest_tools_bindings_stripped(): +async def test_vector_155_wireconformance_torequest_tools_bindings_stripped(): vector_json = r""" { "name": "tools_bindings_stripped", @@ -9594,7 +9624,7 @@ async def test_vector_154_wireconformance_torequest_tools_bindings_stripped(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_155_wireconformance_torequest_tools_function_wire(): +async def test_vector_156_wireconformance_torequest_tools_function_wire(): vector_json = r""" { "name": "tools_function_wire", @@ -9676,7 +9706,7 @@ async def test_vector_155_wireconformance_torequest_tools_function_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_156_wireconformance_torequest_tools_null_when_empty(): +async def test_vector_157_wireconformance_torequest_tools_null_when_empty(): vector_json = r""" { "name": "tools_null_when_empty", @@ -9725,7 +9755,7 @@ async def test_vector_156_wireconformance_torequest_tools_null_when_empty(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_157_wireconformance_torequest_tools_strict_mode(): +async def test_vector_158_wireconformance_torequest_tools_strict_mode(): vector_json = r""" { "name": "tools_strict_mode", diff --git a/runtime/rust/prompty/src/loader/endpoint_trust.rs b/runtime/rust/prompty/src/loader/endpoint_trust.rs index c0b3740ef..bdddcbf46 100644 --- a/runtime/rust/prompty/src/loader/endpoint_trust.rs +++ b/runtime/rust/prompty/src/loader/endpoint_trust.rs @@ -140,7 +140,8 @@ fn file_supplied_credential(connection: &serde_json::Map) -> bool _ => return false, }; - matches!(connection.get(key), Some(Value::String(s)) if !s.is_empty() && !is_reference(s)) + // Blank keys count as absent: executors that trim fall back to ambient credentials. + matches!(connection.get(key), Some(Value::String(s)) if !s.trim().is_empty() && !is_reference(s)) } /// Return the effective destination if the file chose it, else `None`. diff --git a/runtime/rust/prompty/src/model/vector_conformance.rs b/runtime/rust/prompty/src/model/vector_conformance.rs index d8ee20e91..7a2ade8d0 100644 --- a/runtime/rust/prompty/src/model/vector_conformance.rs +++ b/runtime/rust/prompty/src/model/vector_conformance.rs @@ -1631,6 +1631,20 @@ pub async fn run_load_conformance_conformance(_ seam: S) async t throw SeamConformanceError("untrusted_endpoint_backslash_rejected: expected an error") } } + // vector: untrusted_endpoint_blank_literal_key_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-blank-key\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\" \"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_blank_literal_key_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_blank_literal_key_rejected: expected an error") + } + } // vector: untrusted_endpoint_env_default_rejected do { let inputData = "{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}}".data(using: .utf8)! diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift index cf03038ca..da3c7a341 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift @@ -614,7 +614,17 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector58LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() async throws { + func testVector58LoadConformanceLoadUntrustedEndpointBlankLiteralKeyRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_blank_literal_key_rejected\",\"description\":\"a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-blank-key\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\" \"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector59LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -624,7 +634,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector59LoadConformanceLoadUntrustedEndpointEnvKeyRejected() async throws { + func testVector60LoadConformanceLoadUntrustedEndpointEnvKeyRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -634,7 +644,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector60LoadConformanceLoadUntrustedEndpointFileConnectionRejected() async throws { + func testVector61LoadConformanceLoadUntrustedEndpointFileConnectionRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -644,7 +654,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector61LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() async throws { + func testVector62LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_fragment_before_at_rejected\",\"description\":\"A '#' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -654,7 +664,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector62LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() async throws { + func testVector63LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_ipv6_literal_rejected\",\"description\":\"An IPv6 literal is not a trusted origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -664,7 +674,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector63LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() async throws { + func testVector64LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -674,7 +684,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector64LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() async throws { + func testVector65LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -684,7 +694,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector65LoadConformanceLoadUntrustedEndpointNoKeyRejected() async throws { + func testVector66LoadConformanceLoadUntrustedEndpointNoKeyRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -694,7 +704,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector66LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() async throws { + func testVector67LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_oauth_literal_secret_rejected\",\"description\":\"kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -704,7 +714,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector67LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() async throws { + func testVector68LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_percent_encoded_host_rejected\",\"description\":\"Hosts are compared without percent-decoding, so an encoded host never matches\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -714,7 +724,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector68LoadConformanceLoadUntrustedEndpointPortMismatchRejected() async throws { + func testVector69LoadConformanceLoadUntrustedEndpointPortMismatchRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_port_mismatch_rejected\",\"description\":\"A non-default port is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -724,7 +734,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector69LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() async throws { + func testVector70LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_query_before_at_rejected\",\"description\":\"A '?' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -734,7 +744,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector70LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() async throws { + func testVector71LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_scheme_mismatch_rejected\",\"description\":\"Built-in origins are https; http on the same host is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -744,7 +754,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector71LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() async throws { + func testVector72LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_suffix_lookalike_rejected\",\"description\":\"A trusted host as a label prefix of another domain is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -754,7 +764,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector72LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() async throws { + func testVector73LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -764,7 +774,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector73LoadConformanceLoadUntrustedEndpointTrailingDotRejected() async throws { + func testVector74LoadConformanceLoadUntrustedEndpointTrailingDotRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_trailing_dot_rejected\",\"description\":\"Hosts are compared exactly; a trailing-dot host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -774,7 +784,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector74LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() async throws { + func testVector75LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_unicode_homoglyph_rejected\",\"description\":\"A non-ASCII lookalike host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\u{430}pi.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -784,7 +794,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector75LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() async throws { + func testVector76LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_userinfo_lookalike_rejected\",\"description\":\"A trusted host in userinfo is not the destination host\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -794,7 +804,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector76LoadConformanceLoadUntrustedEndpointWildcardApexRejected() async throws { + func testVector77LoadConformanceLoadUntrustedEndpointWildcardApexRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -804,7 +814,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector77LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() async throws { + func testVector78LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() async throws { guard let vectorData = "{\"name\":\"untrusted_endpoint_wildcard_boundary_rejected\",\"description\":\"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -814,7 +824,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector78MemoryConformanceOperateClearByCategoryAndAll() async throws { + func testVector79MemoryConformanceOperateClearByCategoryAndAll() async throws { guard let vectorData = "{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -824,7 +834,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector79MemoryConformanceOperateFormatCoreAndRecallResults() async throws { + func testVector80MemoryConformanceOperateFormatCoreAndRecallResults() async throws { guard let vectorData = "{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -834,7 +844,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector80MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { + func testVector81MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { guard let vectorData = "{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -844,7 +854,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector81MemoryConformanceOperateRecallRanksByWeightedScore() async throws { + func testVector82MemoryConformanceOperateRecallRanksByWeightedScore() async throws { guard let vectorData = "{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -854,7 +864,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector82MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { + func testVector83MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { guard let vectorData = "{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -864,7 +874,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector83MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { + func testVector84MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { guard let vectorData = "{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -874,7 +884,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector84MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { + func testVector85MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { guard let vectorData = "{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\u{e9}\"]},{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\u{e9}\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -884,7 +894,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector85MemoryConformanceOperateRememberDedupsCoreByTags() async throws { + func testVector86MemoryConformanceOperateRememberDedupsCoreByTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -894,7 +904,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector86MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { + func testVector87MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -904,7 +914,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector87MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { + func testVector88MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { guard let vectorData = "{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -914,7 +924,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector88MemoryConformanceOperateSnapshotPortRoundTrip() async throws { + func testVector89MemoryConformanceOperateSnapshotPortRoundTrip() async throws { guard let vectorData = "{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -924,7 +934,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector89MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { + func testVector90MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { guard let vectorData = "{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -934,7 +944,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector90TurnConformanceReplayMaxIterations() async throws { + func testVector91TurnConformanceReplayMaxIterations() async throws { guard let vectorData = "{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -944,7 +954,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector91TurnConformanceReplayNoTool() async throws { + func testVector92TurnConformanceReplayNoTool() async throws { guard let vectorData = "{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -954,7 +964,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector92TurnConformanceReplayPermissionDenied() async throws { + func testVector93TurnConformanceReplayPermissionDenied() async throws { guard let vectorData = "{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -964,7 +974,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector93TurnConformanceReplayToolFailure() async throws { + func testVector94TurnConformanceReplayToolFailure() async throws { guard let vectorData = "{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -974,7 +984,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector94TurnConformanceReplayToolSuccess() async throws { + func testVector95TurnConformanceReplayToolSuccess() async throws { guard let vectorData = "{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -984,7 +994,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector95TurnConformanceRunAssistantToolCallsMetadata() async throws { + func testVector96TurnConformanceRunAssistantToolCallsMetadata() async throws { guard let vectorData = "{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -994,7 +1004,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector96TurnConformanceRunAsyncToolFunction() async throws { + func testVector97TurnConformanceRunAsyncToolFunction() async throws { guard let vectorData = "{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1004,7 +1014,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector97TurnConformanceRunBindingsInjected() async throws { + func testVector98TurnConformanceRunBindingsInjected() async throws { guard let vectorData = "{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1014,7 +1024,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector98TurnConformanceRunCancellationBeforeLlm() async throws { + func testVector99TurnConformanceRunCancellationBeforeLlm() async throws { guard let vectorData = "{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1024,7 +1034,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector99TurnConformanceRunCancellationBetweenIterations() async throws { + func testVector100TurnConformanceRunCancellationBetweenIterations() async throws { guard let vectorData = "{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1034,7 +1044,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector100TurnConformanceRunCancellationBetweenTools() async throws { + func testVector101TurnConformanceRunCancellationBetweenTools() async throws { guard let vectorData = "{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1044,7 +1054,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector101TurnConformanceRunContextNoTrimWhenFits() async throws { + func testVector102TurnConformanceRunContextNoTrimWhenFits() async throws { guard let vectorData = "{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1054,7 +1064,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector102TurnConformanceRunContextPreservesSystemMessages() async throws { + func testVector103TurnConformanceRunContextPreservesSystemMessages() async throws { guard let vectorData = "{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1064,7 +1074,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector103TurnConformanceRunContextTrimBasic() async throws { + func testVector104TurnConformanceRunContextTrimBasic() async throws { guard let vectorData = "{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1074,7 +1084,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector104TurnConformanceRunEmptyToolResult() async throws { + func testVector105TurnConformanceRunEmptyToolResult() async throws { guard let vectorData = "{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1084,7 +1094,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector105TurnConformanceRunEventsBasicToolLoop() async throws { + func testVector106TurnConformanceRunEventsBasicToolLoop() async throws { guard let vectorData = "{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1094,7 +1104,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector106TurnConformanceRunEventsErrorLogged() async throws { + func testVector107TurnConformanceRunEventsErrorLogged() async throws { guard let vectorData = "{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1104,7 +1114,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector107TurnConformanceRunEventsNoTools() async throws { + func testVector108TurnConformanceRunEventsNoTools() async throws { guard let vectorData = "{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1114,7 +1124,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector108TurnConformanceRunGuardrailAllPass() async throws { + func testVector109TurnConformanceRunGuardrailAllPass() async throws { guard let vectorData = "{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1124,7 +1134,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector109TurnConformanceRunGuardrailInputDeny() async throws { + func testVector110TurnConformanceRunGuardrailInputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1134,7 +1144,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector110TurnConformanceRunGuardrailOutputDeny() async throws { + func testVector111TurnConformanceRunGuardrailOutputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1144,7 +1154,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector111TurnConformanceRunGuardrailToolDeny() async throws { + func testVector112TurnConformanceRunGuardrailToolDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1154,7 +1164,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector112TurnConformanceRunMaxIterationsExceeded() async throws { + func testVector113TurnConformanceRunMaxIterationsExceeded() async throws { guard let vectorData = "{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1164,7 +1174,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector113TurnConformanceRunMultiTurnToolCalls() async throws { + func testVector114TurnConformanceRunMultiTurnToolCalls() async throws { guard let vectorData = "{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1174,7 +1184,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector114TurnConformanceRunMultipleToolCallsSingleTurn() async throws { + func testVector115TurnConformanceRunMultipleToolCallsSingleTurn() async throws { guard let vectorData = "{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1184,7 +1194,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector115TurnConformanceRunNoToolCalls() async throws { + func testVector116TurnConformanceRunNoToolCalls() async throws { guard let vectorData = "{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1194,7 +1204,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector116TurnConformanceRunParallelToolsBasic() async throws { + func testVector117TurnConformanceRunParallelToolsBasic() async throws { guard let vectorData = "{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1204,7 +1214,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector117TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { + func testVector118TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1214,7 +1224,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector118TurnConformanceRunParallelToolsWithToolError() async throws { + func testVector119TurnConformanceRunParallelToolsWithToolError() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\u{a7}13.6 Parallel Tools + tool error \u{2014} 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1224,7 +1234,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector119TurnConformanceRunSingleToolCall() async throws { + func testVector120TurnConformanceRunSingleToolCall() async throws { guard let vectorData = "{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1234,7 +1244,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector120TurnConformanceRunSteeringInjectMessage() async throws { + func testVector121TurnConformanceRunSteeringInjectMessage() async throws { guard let vectorData = "{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1244,7 +1254,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector121TurnConformanceRunSteeringMultipleMessages() async throws { + func testVector122TurnConformanceRunSteeringMultipleMessages() async throws { guard let vectorData = "{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1254,7 +1264,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector122TurnConformanceRunToolNotRegisteredError() async throws { + func testVector123TurnConformanceRunToolNotRegisteredError() async throws { guard let vectorData = "{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1264,7 +1274,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector123TurnConformanceRunToolResultMessageFormat() async throws { + func testVector124TurnConformanceRunToolResultMessageFormat() async throws { guard let vectorData = "{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1274,7 +1284,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector124TurnConformanceRunTurnCancelBeforeContext() async throws { + func testVector125TurnConformanceRunTurnCancelBeforeContext() async throws { guard let vectorData = "{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1284,7 +1294,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector125TurnConformanceRunTurnDelegatedProviderState() async throws { + func testVector126TurnConformanceRunTurnDelegatedProviderState() async throws { guard let vectorData = "{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1294,7 +1304,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector126TurnConformanceRunTurnFinalOutput() async throws { + func testVector127TurnConformanceRunTurnFinalOutput() async throws { guard let vectorData = "{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1304,7 +1314,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector127TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { + func testVector128TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { guard let vectorData = "{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1314,7 +1324,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector128TurnConformanceRunTurnOrderedToolRound() async throws { + func testVector129TurnConformanceRunTurnOrderedToolRound() async throws { guard let vectorData = "{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1324,7 +1334,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector129TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { + func testVector130TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { guard let vectorData = "{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1334,7 +1344,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector130WireConformanceToRequestAnthropicImageFormat() async throws { + func testVector131WireConformanceToRequestAnthropicImageFormat() async throws { guard let vectorData = "{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1344,7 +1354,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector131WireConformanceToRequestAnthropicMaxTokensRequired() async throws { + func testVector132WireConformanceToRequestAnthropicMaxTokensRequired() async throws { guard let vectorData = "{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1354,7 +1364,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector132WireConformanceToRequestAnthropicOptions() async throws { + func testVector133WireConformanceToRequestAnthropicOptions() async throws { guard let vectorData = "{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1364,7 +1374,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector133WireConformanceToRequestAnthropicSystemSeparate() async throws { + func testVector134WireConformanceToRequestAnthropicSystemSeparate() async throws { guard let vectorData = "{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1374,7 +1384,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector134WireConformanceToRequestAnthropicToolWire() async throws { + func testVector135WireConformanceToRequestAnthropicToolWire() async throws { guard let vectorData = "{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1384,7 +1394,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector135WireConformanceToRequestChatAudioMp3() async throws { + func testVector136WireConformanceToRequestChatAudioMp3() async throws { guard let vectorData = "{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1394,7 +1404,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector136WireConformanceToRequestChatAudioPart() async throws { + func testVector137WireConformanceToRequestChatAudioPart() async throws { guard let vectorData = "{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1404,7 +1414,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector137WireConformanceToRequestChatImageBase64() async throws { + func testVector138WireConformanceToRequestChatImageBase64() async throws { guard let vectorData = "{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1414,7 +1424,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector138WireConformanceToRequestChatImagePart() async throws { + func testVector139WireConformanceToRequestChatImagePart() async throws { guard let vectorData = "{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1424,7 +1434,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector139WireConformanceToRequestChatMultipartContent() async throws { + func testVector140WireConformanceToRequestChatMultipartContent() async throws { guard let vectorData = "{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1434,7 +1444,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector140WireConformanceToRequestChatSimple() async throws { + func testVector141WireConformanceToRequestChatSimple() async throws { guard let vectorData = "{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1444,7 +1454,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector141WireConformanceToRequestChatSingleTextOptimized() async throws { + func testVector142WireConformanceToRequestChatSingleTextOptimized() async throws { guard let vectorData = "{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1454,7 +1464,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector142WireConformanceToRequestChatWithOptions() async throws { + func testVector143WireConformanceToRequestChatWithOptions() async throws { guard let vectorData = "{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1464,7 +1474,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector143WireConformanceToRequestEmbeddingWire() async throws { + func testVector144WireConformanceToRequestEmbeddingWire() async throws { guard let vectorData = "{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1474,7 +1484,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector144WireConformanceToRequestImageWire() async throws { + func testVector145WireConformanceToRequestImageWire() async throws { guard let vectorData = "{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1484,7 +1494,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector145WireConformanceToRequestKindToJsonTypeMapping() async throws { + func testVector146WireConformanceToRequestKindToJsonTypeMapping() async throws { guard let vectorData = "{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1494,7 +1504,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector146WireConformanceToRequestOptionsAdditionalProperties() async throws { + func testVector147WireConformanceToRequestOptionsAdditionalProperties() async throws { guard let vectorData = "{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1504,7 +1514,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector147WireConformanceToRequestOptionsMaxCompletionTokens() async throws { + func testVector148WireConformanceToRequestOptionsMaxCompletionTokens() async throws { guard let vectorData = "{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1514,7 +1524,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector148WireConformanceToRequestOptionsStopSequences() async throws { + func testVector149WireConformanceToRequestOptionsStopSequences() async throws { guard let vectorData = "{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1524,7 +1534,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector149WireConformanceToRequestResponsesSimple() async throws { + func testVector150WireConformanceToRequestResponsesSimple() async throws { guard let vectorData = "{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1534,7 +1544,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector150WireConformanceToRequestResponsesStructuredOutput() async throws { + func testVector151WireConformanceToRequestResponsesStructuredOutput() async throws { guard let vectorData = "{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1544,7 +1554,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector151WireConformanceToRequestResponsesWithTools() async throws { + func testVector152WireConformanceToRequestResponsesWithTools() async throws { guard let vectorData = "{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1554,7 +1564,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector152WireConformanceToRequestStructuredOutput() async throws { + func testVector153WireConformanceToRequestStructuredOutput() async throws { guard let vectorData = "{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1564,7 +1574,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector153WireConformanceToRequestStructuredOutputNestedOptional() async throws { + func testVector154WireConformanceToRequestStructuredOutputNestedOptional() async throws { guard let vectorData = "{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1574,7 +1584,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector154WireConformanceToRequestToolsBindingsStripped() async throws { + func testVector155WireConformanceToRequestToolsBindingsStripped() async throws { guard let vectorData = "{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1584,7 +1594,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector155WireConformanceToRequestToolsFunctionWire() async throws { + func testVector156WireConformanceToRequestToolsFunctionWire() async throws { guard let vectorData = "{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1594,7 +1604,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector156WireConformanceToRequestToolsNullWhenEmpty() async throws { + func testVector157WireConformanceToRequestToolsNullWhenEmpty() async throws { guard let vectorData = "{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1604,7 +1614,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector157WireConformanceToRequestToolsStrictMode() async throws { + func testVector158WireConformanceToRequestToolsStrictMode() async throws { guard let vectorData = "{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { diff --git a/runtime/typescript/packages/core/src/core/endpoint-trust.ts b/runtime/typescript/packages/core/src/core/endpoint-trust.ts index 1e2d1df24..cb9f00bc3 100644 --- a/runtime/typescript/packages/core/src/core/endpoint-trust.ts +++ b/runtime/typescript/packages/core/src/core/endpoint-trust.ts @@ -103,7 +103,10 @@ function fileSuppliedCredential(connection: Record): boolean { : undefined; if (key === undefined) return false; const value = connection[key]; - return typeof value === "string" && value.length > 0 && !isReference(value); + // Blank keys count as absent: executors that trim fall back to ambient credentials. + return ( + typeof value === "string" && value.trim().length > 0 && !isReference(value) + ); } /** Return the effective destination if the file chose it, else `undefined`. */ diff --git a/runtime/typescript/packages/core/src/model/vector-conformance.ts b/runtime/typescript/packages/core/src/model/vector-conformance.ts index bf46731ad..b6a7f2a1f 100644 --- a/runtime/typescript/packages/core/src/model/vector-conformance.ts +++ b/runtime/typescript/packages/core/src/model/vector-conformance.ts @@ -804,6 +804,21 @@ export async function runLoadConformanceConformance( "untrusted_endpoint_backslash_rejected: expected an error", ); } + // vector: untrusted_endpoint_blank_literal_key_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_blank_literal_key_rejected: expected an error", + ); + } // vector: untrusted_endpoint_env_default_rejected { const sourceJson = "null"; diff --git a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts index 2af293a40..ff862a14f 100644 --- a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts +++ b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts @@ -387,6 +387,12 @@ describe("callable vector conformance", () => { ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:untrusted_endpoint_blank_literal_key_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_blank_literal_key_rejected","description":"a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials","stage":"load","input":{"frontmatter":{"name":"untrusted-blank-key","model":{"id":"gpt-4","provider":"foundry","connection":{"kind":"key","endpoint":"https://attacker.example.com","apiKey":" "}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("LoadConformance.load:untrusted_endpoint_env_default_rejected", async () => { const vector = JSON.parse( '{"name":"untrusted_endpoint_env_default_rejected","description":"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-env-default","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"${env:NONEXISTENT:https://attacker.example.com}"}}},"env":{}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', diff --git a/schema/model/conformance/vectors/load.tsp b/schema/model/conformance/vectors/load.tsp index 1ce087088..1fb6cec47 100644 --- a/schema/model/conformance/vectors/load.tsp +++ b/schema/model/conformance/vectors/load.tsp @@ -377,6 +377,28 @@ const LoadVectors = """ "kind": "untrusted_endpoint" } }, + { + "name": "untrusted_endpoint_blank_literal_key_rejected", + "description": "a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-blank-key", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": " " + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, { "name": "untrusted_endpoint_oauth_literal_secret_rejected", "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", diff --git a/schema/tsp-output/.typra-generated/vectors.json b/schema/tsp-output/.typra-generated/vectors.json index 1ced3bc64..7dfbaa19a 100644 --- a/schema/tsp-output/.typra-generated/vectors.json +++ b/schema/tsp-output/.typra-generated/vectors.json @@ -2228,6 +2228,40 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_blank_literal_key_rejected", + "description": "a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-blank-key", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": " " + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, { "contract": "LoadConformance", "namespace": "Prompty.Conformance.Seams", diff --git a/spec/spec.md b/spec/spec.md index 927e9d6dd..cf25a8965 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -296,7 +296,9 @@ Loaders MUST enforce this at load time (§4.2 step 5) on `model.connection`: 1. Skip the check when `connection` is absent or `kind` is `reference`. 2. The **credential is file-supplied** only when `kind` is `key` and `apiKey` is a - non-empty literal string (not a `${...}` reference). Every other case is ambient, + literal string that is not blank (contains a non-whitespace character) and is not a + `${...}` reference. A blank `apiKey` is ambient because executors treat it as absent + and fall back to ambient credentials. Every other case is ambient, including `${env:}` or `${file:}` keys, any other `kind` (e.g. `foundry`, `oauth`, `anonymous`, custom), and an `apiKey` on any other `kind`. (`oauth` is ambient because executors do not consume `clientSecret` and fall back to ambient credentials.)