diff --git a/runtime/csharp/CHANGELOG.md b/runtime/csharp/CHANGELOG.md index 55a819d0a..646e90a6e 100644 --- a/runtime/csharp/CHANGELOG.md +++ b/runtime/csharp/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to the Prompty C# runtime will be documented here. +## [Unreleased] +### Security +- Loader enforces spec §2.5.1 endpoint trust: a `.prompty` whose `model.connection` would send ambient credentials to a file-chosen, untrusted `endpoint`/`tokenUrl` fails with `untrusted_endpoint`. New `PromptyLoadOptions.TrustedEndpoints` option and `PROMPTY_TRUSTED_ENDPOINTS` env var. + ## [2.0.0] — 2026-09-16 ### Added diff --git a/runtime/csharp/Prompty.Core.Tests/EndpointTrustTests.cs b/runtime/csharp/Prompty.Core.Tests/EndpointTrustTests.cs new file mode 100644 index 000000000..239d665e9 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/EndpointTrustTests.cs @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft. All rights reserved. +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// Tests for endpoint trust (spec §2.5.1). +public sealed class EndpointTrustTests : IDisposable +{ + private readonly string _tmp; + private readonly Dictionary _savedEnv = []; + + public EndpointTrustTests() + { + _tmp = Path.Combine(Path.GetTempPath(), $"prompty-trust-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tmp); + SetEnv(EndpointTrust.TrustedEndpointsEnv, null); + } + + public void Dispose() + { + foreach (var (key, value) in _savedEnv) + Environment.SetEnvironmentVariable(key, value); + try + { + Directory.Delete(_tmp, recursive: true); + } + catch (IOException) + { + } + } + + private void SetEnv(string key, string? value) + { + _savedEnv.TryAdd(key, Environment.GetEnvironmentVariable(key)); + Environment.SetEnvironmentVariable(key, value); + } + + private string Write(string connection) + { + var p = Path.Combine(_tmp, "t.prompty"); + File.WriteAllText(p, $"---\nname: t\nmodel:\n id: gpt-4\n connection:\n{connection}\n---\nHello\n"); + return p; + } + + // --- IsTrustedEndpoint --- + + [Theory] + [InlineData("https://api.openai.com/v1")] + [InlineData("https://API.OpenAI.com:443")] + [InlineData("https://myres.openai.azure.com/")] + [InlineData("https://a.b.services.ai.azure.com/api/projects/p")] + [InlineData("https://login.microsoftonline.com/tenant/oauth2/v2.0/token")] + public void BuiltinTrusted(string url) => Assert.True(EndpointTrust.IsTrustedEndpoint(url)); + + [Theory] + [InlineData("http://api.openai.com")] // scheme mismatch + [InlineData("https://api.openai.com:8443")] // port mismatch + [InlineData("https://openai.azure.com")] // wildcard never matches apex + [InlineData("https://api.openai.com.evil.com")] + [InlineData("https://evilopenai.azure.com")] + [InlineData("https://api.openai.com@evil.com")] // userinfo ignored; host is evil.com + [InlineData("http://localhost:11434")] + [InlineData("not a url")] + [InlineData("ftp://api.openai.com")] + [InlineData("")] + [InlineData("https://api.openai.com:99999")] // invalid port + [InlineData("https://api.openai.com:abc")] + [InlineData("https://[::1")] + public void Untrusted(string url) => Assert.False(EndpointTrust.IsTrustedEndpoint(url)); + + [Fact] + public void HostOptionAndWildcard() + { + Assert.True(EndpointTrust.IsTrustedEndpoint("http://localhost:11434/v1", ["http://LOCALHOST:11434"])); + Assert.True(EndpointTrust.IsTrustedEndpoint("https://x.contoso.com", ["https://*.contoso.com"])); + Assert.False(EndpointTrust.IsTrustedEndpoint("https://contoso.com", ["https://*.contoso.com"])); + Assert.True(EndpointTrust.IsTrustedEndpoint("http://[::1]:8080/x", ["http://[::1]:8080"])); + } + + [Fact] + public void EnvListAndStar() + { + SetEnv(EndpointTrust.TrustedEndpointsEnv, " https://a.example.com , http://localhost:8080"); + Assert.True(EndpointTrust.IsTrustedEndpoint("http://localhost:8080/x")); + Assert.False(EndpointTrust.IsTrustedEndpoint("http://localhost:8081")); + SetEnv(EndpointTrust.TrustedEndpointsEnv, "*"); + Assert.True(EndpointTrust.IsTrustedEndpoint("http://anything.example")); + } + + // --- Loader enforcement --- + + [Fact] + public void AmbientLiteralEndpointRejected() + { + var p = Write(" kind: foundry\n endpoint: http://127.0.0.1:9999"); + var ex = Assert.Throws(() => PromptyLoader.Load(p)); + Assert.Equal("untrusted_endpoint", ex.Kind); + Assert.Contains("127.0.0.1", ex.Message); + } + + [Fact] + public void EnvKeyLiteralEndpointRejected() + { + SetEnv("SOME_KEY", "secret"); + var p = Write(" kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}"); + var ex = Assert.Throws(() => PromptyLoader.Load(p)); + Assert.Contains("not trusted", ex.Message); + } + + [Fact] + public void OAuthTokenUrlChecked() + { + SetEnv("SECRET", "s"); + var p = Write( + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token"); + var ex = Assert.Throws(() => PromptyLoader.Load(p)); + Assert.Contains("tokenUrl", ex.Message); + } + + [Fact] + public void MismatchedCredentialFieldRejected() + { + var p = Write(" kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk"); + Assert.Contains("not trusted", Assert.Throws(() => PromptyLoader.Load(p)).Message); + p = Write(" kind: key\n endpoint: https://evil.example.com\n clientSecret: junk"); + Assert.Contains("not trusted", Assert.Throws(() => PromptyLoader.Load(p)).Message); + } + + [Fact] + public void LiteralKeyAllowed() + { + var agent = PromptyLoader.Load(Write(" kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine")); + Assert.Equal("http://127.0.0.1:9999", Assert.IsType(agent.Model!.Connection).Endpoint); + } + + [Fact] + public void EnvEndpointAllowed() + { + SetEnv("EP", "http://127.0.0.1:9999"); + var agent = PromptyLoader.Load(Write(" kind: foundry\n endpoint: ${env:EP}")); + Assert.Equal("http://127.0.0.1:9999", Assert.IsType(agent.Model!.Connection).Endpoint); + } + + [Fact] + public void EnvDefaultUsedRejected() + { + SetEnv("UNSET_EP", null); + var p = Write(" kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}"); + Assert.Contains("not trusted", Assert.Throws(() => PromptyLoader.Load(p)).Message); + } + + [Fact] + public void TrustedOptionAllows() + { + var p = Write(" kind: foundry\n endpoint: http://127.0.0.1:9999/api"); + var agent = PromptyLoader.Load(p, new PromptyLoadOptions { TrustedEndpoints = ["http://127.0.0.1:9999"] }); + Assert.Equal("http://127.0.0.1:9999/api", Assert.IsType(agent.Model!.Connection).Endpoint); + } + + [Fact] + public async Task AsyncHonorsOption() + { + var p = Write(" kind: foundry\n endpoint: http://127.0.0.1:9999"); + await Assert.ThrowsAsync(() => PromptyLoader.LoadAsync(p)); + var agent = await PromptyLoader.LoadAsync( + p, new PromptyLoadOptions { TrustedEndpoints = ["http://127.0.0.1:9999"] }); + Assert.Equal("http://127.0.0.1:9999", Assert.IsType(agent.Model!.Connection).Endpoint); + } + + [Fact] + public void FileModelReferenceChecked() + { + File.WriteAllText( + Path.Combine(_tmp, "model.json"), + """{"id": "gpt-4", "connection": {"kind": "foundry", "endpoint": "https://evil.example.com"}}"""); + var p = Path.Combine(_tmp, "t.prompty"); + File.WriteAllText(p, "---\nname: t\nmodel: ${file:model.json}\n---\nHi\n"); + Assert.Contains("not trusted", Assert.Throws(() => PromptyLoader.Load(p)).Message); + } + + [Fact] + public void FileConnectionReferenceChecked() + { + File.WriteAllText(Path.Combine(_tmp, "conn.yaml"), "kind: foundry\nendpoint: https://evil.example.com\n"); + var p = Path.Combine(_tmp, "t.prompty"); + File.WriteAllText(p, "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:conn.yaml}\n---\nHi\n"); + var ex = Assert.Throws(() => PromptyLoader.Load(p)); + Assert.Equal("untrusted_endpoint", ex.Kind); + } + + [Fact] + public void FrontmatterCannotSelfTrust() + { + var p = Path.Combine(_tmp, "t.prompty"); + File.WriteAllText( + p, + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n"); + Assert.Throws(() => PromptyLoader.Load(p)); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs index 4f82cd411..0e95c5c84 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorAdapters.cs @@ -296,6 +296,19 @@ private static List LiveProviderMessages(Dictionary fl Environment.SetEnvironmentVariable("NONEXISTENT", null); } + // Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + if (input["env"] is not JsonObject envVars || !envVars.ContainsKey(EndpointTrust.TrustedEndpointsEnv)) + { + savedEnv.TryAdd(EndpointTrust.TrustedEndpointsEnv, Environment.GetEnvironmentVariable(EndpointTrust.TrustedEndpointsEnv)); + Environment.SetEnvironmentVariable(EndpointTrust.TrustedEndpointsEnv, null); + } + var loadOptions = new PromptyLoadOptions + { + TrustedEndpoints = input["trustedEndpoints"] is JsonArray trusted + ? trusted.Select(t => t?.GetValue() ?? string.Empty).ToList() + : [], + }; + try { // --- input-validation vectors: BOTH a top-level `inputs` map AND `frontmatter` --- @@ -325,7 +338,7 @@ private static List LiveProviderMessages(Dictionary fl Agent loaded; if (input["fixture"] is JsonValue fixtureVal) { - loaded = PromptyLoader.Load(Path.Combine(SpecFixtures, fixtureVal.GetValue())); + loaded = PromptyLoader.Load(Path.Combine(SpecFixtures, fixtureVal.GetValue()), loadOptions); } else if (input["frontmatter_raw"] is JsonValue rawVal) { @@ -334,7 +347,7 @@ private static List LiveProviderMessages(Dictionary fl { var p = Path.Combine(tmp, "vector.prompty"); File.WriteAllText(p, rawVal.GetValue()); - loaded = PromptyLoader.Load(p); + loaded = PromptyLoader.Load(p, loadOptions); } finally { @@ -343,7 +356,7 @@ private static List LiveProviderMessages(Dictionary fl } else if (input["frontmatter"] is JsonObject) { - loaded = MaterializeAndLoad(input); + loaded = MaterializeAndLoad(input, loadOptions); } else { @@ -436,7 +449,7 @@ private static Agent MakeAgentFromFrontmatter(JsonNode? frontmatter) /// .. are written relative to the agent directory so path-traversal vectors /// can place a target outside the allowed root; the loader must reject the escape. /// - private static Agent MaterializeAndLoad(JsonObject input) + private static Agent MaterializeAndLoad(JsonObject input, PromptyLoadOptions? options = null) { var tempBase = NewTempDir(); try @@ -464,7 +477,7 @@ private static Agent MaterializeAndLoad(JsonObject input) var frontmatter = input["frontmatter"]!.ToJsonString(); var agentPath = Path.Join(agentDir, "vector.prompty"); File.WriteAllText(agentPath, $"---\n{frontmatter}\n---\n"); - return PromptyLoader.Load(agentPath); + return PromptyLoader.Load(agentPath, options); } finally { diff --git a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs index 0a7740e7c..d3aba3b54 100644 --- a/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/Model/VectorConformanceTests.cs @@ -828,7 +828,10 @@ public async Task Vector21LoadConformanceLoadConnectionTypesLoad() "endpoint": "https://localhost:8080" } } - } + }, + "trustedEndpoints": [ + "https://localhost:8080" + ] }, "expected": { "model": { @@ -931,7 +934,8 @@ public async Task Vector24LoadConformanceLoadEnvDefault() "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "${env:MISSING_VAR:fallback_value}" + "endpoint": "${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }, @@ -1793,7 +1797,1007 @@ public async Task Vector47LoadConformanceLoadToolsOpenapiLoad() } [Fact] - public async Task Vector48MemoryConformanceOperateClearByCategoryAndAll() + public async Task Vector48LoadConformanceLoadTrustedEndpointBuiltinAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_builtin_allowed", + "description": "Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-builtin", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector49LoadConformanceLoadTrustedEndpointDefaultPortAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_default_port_allowed", + "description": "An explicit default port normalizes to the built-in origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-default-port-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector50LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_env_endpoint_allowed", + "description": "An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-endpoint", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:VECTOR_FOUNDRY_ENDPOINT}" + } + } + }, + "env": { + "VECTOR_FOUNDRY_ENDPOINT": "https://host-chosen.example.com" + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://host-chosen.example.com" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector51LoadConformanceLoadTrustedEndpointEnvListAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_env_list_allowed", + "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-list", + "model": { + "id": "llama3", + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "env": { + "PROMPTY_TRUSTED_ENDPOINTS": "https://other.example.com, http://localhost:11434" + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector52LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_literal_key_allowed", + "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-literal-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com", + "apiKey": "file-supplied-key" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector53LoadConformanceLoadTrustedEndpointOptionAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_option_allowed", + "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-option", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "trustedEndpoints": [ + "HTTPS://PROXY.Example.com" + ] + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector54LoadConformanceLoadTrustedEndpointReferenceSkipped() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_reference_skipped", + "description": "kind: reference connections carry no endpoint and are not subject to the trust check", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-reference", + "model": { + "id": "gpt-4", + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector55LoadConformanceLoadTrustedEndpointUserinfoAllowed() + { + string vectorJson = """ +{ + "name": "trusted_endpoint_userinfo_allowed", + "description": "Userinfo is ignored; the destination host api.openai.com is trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-userinfo-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector56LoadConformanceLoadUntrustedEndpointAmbientRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_ambient_rejected", + "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec \u00a72.5.1)", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ambient", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com/api/projects/p" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector57LoadConformanceLoadUntrustedEndpointBackslashRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_backslash_rejected", + "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-backslash", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com\\@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector58LoadConformanceLoadUntrustedEndpointBlankLiteralKeyRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_blank_literal_key_rejected", + "description": "a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-blank-key", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": " " + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector59LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_env_default_rejected", + "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-default", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:NONEXISTENT:https://attacker.example.com}" + } + } + }, + "env": {} + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector60LoadConformanceLoadUntrustedEndpointEnvKeyRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_env_key_rejected", + "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": "${env:VECTOR_HOST_KEY}" + } + } + }, + "env": { + "VECTOR_HOST_KEY": "host-secret" + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector61LoadConformanceLoadUntrustedEndpointFileConnectionRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_file_connection_rejected", + "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-file-conn", + "model": { + "id": "gpt-4", + "connection": "${file:conn.json}" + } + }, + "files": { + "conn.json": { + "kind": "foundry", + "endpoint": "https://attacker.example.com" + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector62LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_fragment_before_at_rejected", + "description": "A '#' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-fragment-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com#@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector63LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_ipv6_literal_rejected", + "description": "An IPv6 literal is not a trusted origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ipv6-literal", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://[::1]:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector64LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_key_with_client_secret_rejected", + "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-key-client-secret", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "clientSecret": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector65LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_mismatched_credential_rejected", + "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-mismatched-credential", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com", + "apiKey": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector66LoadConformanceLoadUntrustedEndpointNoKeyRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_no_key_rejected", + "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-no-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "http://127.0.0.1:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector67LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_oauth_literal_secret_rejected", + "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-oauth-literal", + "model": { + "id": "gpt-4", + "provider": "anthropic", + "connection": { + "kind": "oauth", + "endpoint": "https://attacker.example.com", + "clientId": "decoy", + "clientSecret": "decoy", + "tokenUrl": "https://login.microsoftonline.com/t/oauth2/v2.0/token" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector68LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_percent_encoded_host_rejected", + "description": "Hosts are compared without percent-decoding, so an encoded host never matches", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-percent-encoded-host", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com%2eevil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector69LoadConformanceLoadUntrustedEndpointPortMismatchRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_port_mismatch_rejected", + "description": "A non-default port is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-port-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com:8443" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector70LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_query_before_at_rejected", + "description": "A '?' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-query-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com?@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector71LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_scheme_mismatch_rejected", + "description": "Built-in origins are https; http on the same host is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-scheme-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector72LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_suffix_lookalike_rejected", + "description": "A trusted host as a label prefix of another domain is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-suffix-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com.evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector73LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_suffix_spoof_rejected", + "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-spoof", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://good.example.com.attacker.example.com" + } + } + }, + "trustedEndpoints": [ + "https://good.example.com" + ] + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector74LoadConformanceLoadUntrustedEndpointTrailingDotRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_trailing_dot_rejected", + "description": "Hosts are compared exactly; a trailing-dot host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-trailing-dot", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com." + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector75LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_unicode_homoglyph_rejected", + "description": "A non-ASCII lookalike host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-unicode-homoglyph", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://\u0430pi.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector76LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_userinfo_lookalike_rejected", + "description": "A trusted host in userinfo is not the destination host", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-userinfo-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com@evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector77LoadConformanceLoadUntrustedEndpointWildcardApexRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_wildcard_apex_rejected", + "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-apex", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://myres.openai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector78LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() + { + string vectorJson = """ +{ + "name": "untrusted_endpoint_wildcard_boundary_rejected", + "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-wildcard-boundary", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evilservices.ai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +} +"""; + var vector = JsonNode.Parse(vectorJson) as JsonObject ?? new JsonObject(); + await VectorRunner.RunVector("LoadConformance", "load", vector, false, Seam()); + } + + [Fact] + public async Task Vector79MemoryConformanceOperateClearByCategoryAndAll() { string vectorJson = """ { @@ -1847,7 +2851,7 @@ public async Task Vector48MemoryConformanceOperateClearByCategoryAndAll() } [Fact] - public async Task Vector49MemoryConformanceOperateFormatCoreAndRecallResults() + public async Task Vector80MemoryConformanceOperateFormatCoreAndRecallResults() { string vectorJson = """ { @@ -1889,7 +2893,7 @@ public async Task Vector49MemoryConformanceOperateFormatCoreAndRecallResults() } [Fact] - public async Task Vector50MemoryConformanceOperateRecallEmptyQueryAndLimit() + public async Task Vector81MemoryConformanceOperateRecallEmptyQueryAndLimit() { string vectorJson = """ { @@ -1944,7 +2948,7 @@ public async Task Vector50MemoryConformanceOperateRecallEmptyQueryAndLimit() } [Fact] - public async Task Vector51MemoryConformanceOperateRecallRanksByWeightedScore() + public async Task Vector82MemoryConformanceOperateRecallRanksByWeightedScore() { string vectorJson = """ { @@ -2003,7 +3007,7 @@ public async Task Vector51MemoryConformanceOperateRecallRanksByWeightedScore() } [Fact] - public async Task Vector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() + public async Task Vector83MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() { string vectorJson = """ { @@ -2061,7 +3065,7 @@ public async Task Vector52MemoryConformanceOperateRecallScoreTiesPreserveInserti } [Fact] - public async Task Vector53MemoryConformanceOperateRecallTagsAndCoreBoost() + public async Task Vector84MemoryConformanceOperateRecallTagsAndCoreBoost() { string vectorJson = """ { @@ -2122,7 +3126,7 @@ public async Task Vector53MemoryConformanceOperateRecallTagsAndCoreBoost() } [Fact] - public async Task Vector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() + public async Task Vector85MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() { string vectorJson = """ { @@ -2183,7 +3187,7 @@ public async Task Vector54MemoryConformanceOperateRecallTokenizationPunctuationU } [Fact] - public async Task Vector55MemoryConformanceOperateRememberDedupsCoreByTags() + public async Task Vector86MemoryConformanceOperateRememberDedupsCoreByTags() { string vectorJson = """ { @@ -2247,7 +3251,7 @@ public async Task Vector55MemoryConformanceOperateRememberDedupsCoreByTags() } [Fact] - public async Task Vector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() + public async Task Vector87MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() { string vectorJson = """ { @@ -2303,7 +3307,7 @@ public async Task Vector56MemoryConformanceOperateRememberDedupsCoreMissingAndEm } [Fact] - public async Task Vector57MemoryConformanceOperateRememberEvictsArchivalFirst() + public async Task Vector88MemoryConformanceOperateRememberEvictsArchivalFirst() { string vectorJson = """ { @@ -2364,7 +3368,7 @@ public async Task Vector57MemoryConformanceOperateRememberEvictsArchivalFirst() } [Fact] - public async Task Vector58MemoryConformanceOperateSnapshotPortRoundTrip() + public async Task Vector89MemoryConformanceOperateSnapshotPortRoundTrip() { string vectorJson = """ { @@ -2404,7 +3408,7 @@ public async Task Vector58MemoryConformanceOperateSnapshotPortRoundTrip() } [Fact] - public async Task Vector59MemoryConformanceOperateUpdateIndexOutOfRangeError() + public async Task Vector90MemoryConformanceOperateUpdateIndexOutOfRangeError() { string vectorJson = """ { @@ -2438,7 +3442,7 @@ public async Task Vector59MemoryConformanceOperateUpdateIndexOutOfRangeError() } [Fact] - public async Task Vector60TurnConformanceReplayMaxIterations() + public async Task Vector91TurnConformanceReplayMaxIterations() { string vectorJson = """ { @@ -2475,7 +3479,7 @@ public async Task Vector60TurnConformanceReplayMaxIterations() } [Fact] - public async Task Vector61TurnConformanceReplayNoTool() + public async Task Vector92TurnConformanceReplayNoTool() { string vectorJson = """ { @@ -2508,7 +3512,7 @@ public async Task Vector61TurnConformanceReplayNoTool() } [Fact] - public async Task Vector62TurnConformanceReplayPermissionDenied() + public async Task Vector93TurnConformanceReplayPermissionDenied() { string vectorJson = """ { @@ -2544,7 +3548,7 @@ public async Task Vector62TurnConformanceReplayPermissionDenied() } [Fact] - public async Task Vector63TurnConformanceReplayToolFailure() + public async Task Vector94TurnConformanceReplayToolFailure() { string vectorJson = """ { @@ -2582,7 +3586,7 @@ public async Task Vector63TurnConformanceReplayToolFailure() } [Fact] - public async Task Vector64TurnConformanceReplayToolSuccess() + public async Task Vector95TurnConformanceReplayToolSuccess() { string vectorJson = """ { @@ -2620,7 +3624,7 @@ public async Task Vector64TurnConformanceReplayToolSuccess() } [Fact] - public async Task Vector65TurnConformanceRunAssistantToolCallsMetadata() + public async Task Vector96TurnConformanceRunAssistantToolCallsMetadata() { string vectorJson = """ { @@ -2774,7 +3778,7 @@ public async Task Vector65TurnConformanceRunAssistantToolCallsMetadata() } [Fact] - public async Task Vector66TurnConformanceRunAsyncToolFunction() + public async Task Vector97TurnConformanceRunAsyncToolFunction() { string vectorJson = """ { @@ -2886,7 +3890,7 @@ public async Task Vector66TurnConformanceRunAsyncToolFunction() } [Fact] - public async Task Vector67TurnConformanceRunBindingsInjected() + public async Task Vector98TurnConformanceRunBindingsInjected() { string vectorJson = """ { @@ -3018,7 +4022,7 @@ public async Task Vector67TurnConformanceRunBindingsInjected() } [Fact] - public async Task Vector68TurnConformanceRunCancellationBeforeLlm() + public async Task Vector99TurnConformanceRunCancellationBeforeLlm() { string vectorJson = """ { @@ -3081,7 +4085,7 @@ public async Task Vector68TurnConformanceRunCancellationBeforeLlm() } [Fact] - public async Task Vector69TurnConformanceRunCancellationBetweenIterations() + public async Task Vector100TurnConformanceRunCancellationBetweenIterations() { string vectorJson = """ { @@ -3213,7 +4217,7 @@ public async Task Vector69TurnConformanceRunCancellationBetweenIterations() } [Fact] - public async Task Vector70TurnConformanceRunCancellationBetweenTools() + public async Task Vector101TurnConformanceRunCancellationBetweenTools() { string vectorJson = """ { @@ -3354,7 +4358,7 @@ public async Task Vector70TurnConformanceRunCancellationBetweenTools() } [Fact] - public async Task Vector71TurnConformanceRunContextNoTrimWhenFits() + public async Task Vector102TurnConformanceRunContextNoTrimWhenFits() { string vectorJson = """ { @@ -3427,7 +4431,7 @@ public async Task Vector71TurnConformanceRunContextNoTrimWhenFits() } [Fact] - public async Task Vector72TurnConformanceRunContextPreservesSystemMessages() + public async Task Vector103TurnConformanceRunContextPreservesSystemMessages() { string vectorJson = """ { @@ -3536,7 +4540,7 @@ public async Task Vector72TurnConformanceRunContextPreservesSystemMessages() } [Fact] - public async Task Vector73TurnConformanceRunContextTrimBasic() + public async Task Vector104TurnConformanceRunContextTrimBasic() { string vectorJson = """ { @@ -3696,7 +4700,7 @@ public async Task Vector73TurnConformanceRunContextTrimBasic() } [Fact] - public async Task Vector74TurnConformanceRunEmptyToolResult() + public async Task Vector105TurnConformanceRunEmptyToolResult() { string vectorJson = """ { @@ -3813,7 +4817,7 @@ public async Task Vector74TurnConformanceRunEmptyToolResult() } [Fact] - public async Task Vector75TurnConformanceRunEventsBasicToolLoop() + public async Task Vector106TurnConformanceRunEventsBasicToolLoop() { string vectorJson = """ { @@ -3961,7 +4965,7 @@ public async Task Vector75TurnConformanceRunEventsBasicToolLoop() } [Fact] - public async Task Vector76TurnConformanceRunEventsErrorLogged() + public async Task Vector107TurnConformanceRunEventsErrorLogged() { string vectorJson = """ { @@ -4089,7 +5093,7 @@ public async Task Vector76TurnConformanceRunEventsErrorLogged() } [Fact] - public async Task Vector77TurnConformanceRunEventsNoTools() + public async Task Vector108TurnConformanceRunEventsNoTools() { string vectorJson = """ { @@ -4169,7 +5173,7 @@ public async Task Vector77TurnConformanceRunEventsNoTools() } [Fact] - public async Task Vector78TurnConformanceRunGuardrailAllPass() + public async Task Vector109TurnConformanceRunGuardrailAllPass() { string vectorJson = """ { @@ -4294,7 +5298,7 @@ public async Task Vector78TurnConformanceRunGuardrailAllPass() } [Fact] - public async Task Vector79TurnConformanceRunGuardrailInputDeny() + public async Task Vector110TurnConformanceRunGuardrailInputDeny() { string vectorJson = """ { @@ -4352,7 +5356,7 @@ public async Task Vector79TurnConformanceRunGuardrailInputDeny() } [Fact] - public async Task Vector80TurnConformanceRunGuardrailOutputDeny() + public async Task Vector111TurnConformanceRunGuardrailOutputDeny() { string vectorJson = """ { @@ -4432,7 +5436,7 @@ public async Task Vector80TurnConformanceRunGuardrailOutputDeny() } [Fact] - public async Task Vector81TurnConformanceRunGuardrailToolDeny() + public async Task Vector112TurnConformanceRunGuardrailToolDeny() { string vectorJson = """ { @@ -4588,7 +5592,7 @@ public async Task Vector81TurnConformanceRunGuardrailToolDeny() } [Fact] - public async Task Vector82TurnConformanceRunMaxIterationsExceeded() + public async Task Vector113TurnConformanceRunMaxIterationsExceeded() { string vectorJson = """ { @@ -5100,7 +6104,7 @@ public async Task Vector82TurnConformanceRunMaxIterationsExceeded() } [Fact] - public async Task Vector83TurnConformanceRunMultiTurnToolCalls() + public async Task Vector114TurnConformanceRunMultiTurnToolCalls() { string vectorJson = """ { @@ -5279,7 +6283,7 @@ public async Task Vector83TurnConformanceRunMultiTurnToolCalls() } [Fact] - public async Task Vector84TurnConformanceRunMultipleToolCallsSingleTurn() + public async Task Vector115TurnConformanceRunMultipleToolCallsSingleTurn() { string vectorJson = """ { @@ -5462,7 +6466,7 @@ public async Task Vector84TurnConformanceRunMultipleToolCallsSingleTurn() } [Fact] - public async Task Vector85TurnConformanceRunNoToolCalls() + public async Task Vector116TurnConformanceRunNoToolCalls() { string vectorJson = """ { @@ -5532,7 +6536,7 @@ public async Task Vector85TurnConformanceRunNoToolCalls() } [Fact] - public async Task Vector86TurnConformanceRunParallelToolsBasic() + public async Task Vector117TurnConformanceRunParallelToolsBasic() { string vectorJson = """ { @@ -5707,7 +6711,7 @@ public async Task Vector86TurnConformanceRunParallelToolsBasic() } [Fact] - public async Task Vector87TurnConformanceRunParallelToolsWithGuardrailDeny() + public async Task Vector118TurnConformanceRunParallelToolsWithGuardrailDeny() { string vectorJson = """ { @@ -5900,7 +6904,7 @@ public async Task Vector87TurnConformanceRunParallelToolsWithGuardrailDeny() } [Fact] - public async Task Vector88TurnConformanceRunParallelToolsWithToolError() + public async Task Vector119TurnConformanceRunParallelToolsWithToolError() { string vectorJson = """ { @@ -6075,7 +7079,7 @@ public async Task Vector88TurnConformanceRunParallelToolsWithToolError() } [Fact] - public async Task Vector89TurnConformanceRunSingleToolCall() + public async Task Vector120TurnConformanceRunSingleToolCall() { string vectorJson = """ { @@ -6224,7 +7228,7 @@ public async Task Vector89TurnConformanceRunSingleToolCall() } [Fact] - public async Task Vector90TurnConformanceRunSteeringInjectMessage() + public async Task Vector121TurnConformanceRunSteeringInjectMessage() { string vectorJson = """ { @@ -6434,7 +7438,7 @@ public async Task Vector90TurnConformanceRunSteeringInjectMessage() } [Fact] - public async Task Vector91TurnConformanceRunSteeringMultipleMessages() + public async Task Vector122TurnConformanceRunSteeringMultipleMessages() { string vectorJson = """ { @@ -6606,7 +7610,7 @@ public async Task Vector91TurnConformanceRunSteeringMultipleMessages() } [Fact] - public async Task Vector92TurnConformanceRunToolNotRegisteredError() + public async Task Vector123TurnConformanceRunToolNotRegisteredError() { string vectorJson = """ { @@ -6692,7 +7696,7 @@ public async Task Vector92TurnConformanceRunToolNotRegisteredError() } [Fact] - public async Task Vector93TurnConformanceRunToolResultMessageFormat() + public async Task Vector124TurnConformanceRunToolResultMessageFormat() { string vectorJson = """ { @@ -6815,7 +7819,7 @@ public async Task Vector93TurnConformanceRunToolResultMessageFormat() } [Fact] - public async Task Vector94TurnConformanceRunTurnCancelBeforeContext() + public async Task Vector125TurnConformanceRunTurnCancelBeforeContext() { string vectorJson = """ { @@ -6849,7 +7853,7 @@ public async Task Vector94TurnConformanceRunTurnCancelBeforeContext() } [Fact] - public async Task Vector95TurnConformanceRunTurnDelegatedProviderState() + public async Task Vector126TurnConformanceRunTurnDelegatedProviderState() { string vectorJson = """ { @@ -6918,7 +7922,7 @@ public async Task Vector95TurnConformanceRunTurnDelegatedProviderState() } [Fact] - public async Task Vector96TurnConformanceRunTurnFinalOutput() + public async Task Vector127TurnConformanceRunTurnFinalOutput() { string vectorJson = """ { @@ -6965,7 +7969,7 @@ public async Task Vector96TurnConformanceRunTurnFinalOutput() } [Fact] - public async Task Vector97TurnConformanceRunTurnMemoryContextIsModelVisible() + public async Task Vector128TurnConformanceRunTurnMemoryContextIsModelVisible() { string vectorJson = """ { @@ -7029,7 +8033,7 @@ public async Task Vector97TurnConformanceRunTurnMemoryContextIsModelVisible() } [Fact] - public async Task Vector98TurnConformanceRunTurnOrderedToolRound() + public async Task Vector129TurnConformanceRunTurnOrderedToolRound() { string vectorJson = """ { @@ -7122,7 +8126,7 @@ public async Task Vector98TurnConformanceRunTurnOrderedToolRound() } [Fact] - public async Task Vector99TurnConformanceRunTurnPermissionDenialIsModelVisible() + public async Task Vector130TurnConformanceRunTurnPermissionDenialIsModelVisible() { string vectorJson = """ { @@ -7175,7 +8179,7 @@ public async Task Vector99TurnConformanceRunTurnPermissionDenialIsModelVisible() } [Fact] - public async Task Vector100WireConformanceToRequestAnthropicImageFormat() + public async Task Vector131WireConformanceToRequestAnthropicImageFormat() { string vectorJson = """ { @@ -7247,7 +8251,7 @@ public async Task Vector100WireConformanceToRequestAnthropicImageFormat() } [Fact] - public async Task Vector101WireConformanceToRequestAnthropicMaxTokensRequired() + public async Task Vector132WireConformanceToRequestAnthropicMaxTokensRequired() { string vectorJson = """ { @@ -7304,7 +8308,7 @@ public async Task Vector101WireConformanceToRequestAnthropicMaxTokensRequired() } [Fact] - public async Task Vector102WireConformanceToRequestAnthropicOptions() + public async Task Vector133WireConformanceToRequestAnthropicOptions() { string vectorJson = """ { @@ -7375,7 +8379,7 @@ public async Task Vector102WireConformanceToRequestAnthropicOptions() } [Fact] - public async Task Vector103WireConformanceToRequestAnthropicSystemSeparate() + public async Task Vector134WireConformanceToRequestAnthropicSystemSeparate() { string vectorJson = """ { @@ -7444,7 +8448,7 @@ public async Task Vector103WireConformanceToRequestAnthropicSystemSeparate() } [Fact] - public async Task Vector104WireConformanceToRequestAnthropicToolWire() + public async Task Vector135WireConformanceToRequestAnthropicToolWire() { string vectorJson = """ { @@ -7531,7 +8535,7 @@ public async Task Vector104WireConformanceToRequestAnthropicToolWire() } [Fact] - public async Task Vector105WireConformanceToRequestChatAudioMp3() + public async Task Vector136WireConformanceToRequestChatAudioMp3() { string vectorJson = """ { @@ -7591,7 +8595,7 @@ public async Task Vector105WireConformanceToRequestChatAudioMp3() } [Fact] - public async Task Vector106WireConformanceToRequestChatAudioPart() + public async Task Vector137WireConformanceToRequestChatAudioPart() { string vectorJson = """ { @@ -7651,7 +8655,7 @@ public async Task Vector106WireConformanceToRequestChatAudioPart() } [Fact] - public async Task Vector107WireConformanceToRequestChatImageBase64() + public async Task Vector138WireConformanceToRequestChatImageBase64() { string vectorJson = """ { @@ -7717,7 +8721,7 @@ public async Task Vector107WireConformanceToRequestChatImageBase64() } [Fact] - public async Task Vector108WireConformanceToRequestChatImagePart() + public async Task Vector139WireConformanceToRequestChatImagePart() { string vectorJson = """ { @@ -7783,7 +8787,7 @@ public async Task Vector108WireConformanceToRequestChatImagePart() } [Fact] - public async Task Vector109WireConformanceToRequestChatMultipartContent() + public async Task Vector140WireConformanceToRequestChatMultipartContent() { string vectorJson = """ { @@ -7849,7 +8853,7 @@ public async Task Vector109WireConformanceToRequestChatMultipartContent() } [Fact] - public async Task Vector110WireConformanceToRequestChatSimple() + public async Task Vector141WireConformanceToRequestChatSimple() { string vectorJson = """ { @@ -7913,7 +8917,7 @@ public async Task Vector110WireConformanceToRequestChatSimple() } [Fact] - public async Task Vector111WireConformanceToRequestChatSingleTextOptimized() + public async Task Vector142WireConformanceToRequestChatSingleTextOptimized() { string vectorJson = """ { @@ -7964,7 +8968,7 @@ public async Task Vector111WireConformanceToRequestChatSingleTextOptimized() } [Fact] - public async Task Vector112WireConformanceToRequestChatWithOptions() + public async Task Vector143WireConformanceToRequestChatWithOptions() { string vectorJson = """ { @@ -8034,7 +9038,7 @@ public async Task Vector112WireConformanceToRequestChatWithOptions() } [Fact] - public async Task Vector113WireConformanceToRequestEmbeddingWire() + public async Task Vector144WireConformanceToRequestEmbeddingWire() { string vectorJson = """ { @@ -8080,7 +9084,7 @@ public async Task Vector113WireConformanceToRequestEmbeddingWire() } [Fact] - public async Task Vector114WireConformanceToRequestImageWire() + public async Task Vector145WireConformanceToRequestImageWire() { string vectorJson = """ { @@ -8126,7 +9130,7 @@ public async Task Vector114WireConformanceToRequestImageWire() } [Fact] - public async Task Vector115WireConformanceToRequestKindToJsonTypeMapping() + public async Task Vector146WireConformanceToRequestKindToJsonTypeMapping() { string vectorJson = """ { @@ -8247,7 +9251,7 @@ public async Task Vector115WireConformanceToRequestKindToJsonTypeMapping() } [Fact] - public async Task Vector116WireConformanceToRequestOptionsAdditionalProperties() + public async Task Vector147WireConformanceToRequestOptionsAdditionalProperties() { string vectorJson = """ { @@ -8303,7 +9307,7 @@ public async Task Vector116WireConformanceToRequestOptionsAdditionalProperties() } [Fact] - public async Task Vector117WireConformanceToRequestOptionsMaxCompletionTokens() + public async Task Vector148WireConformanceToRequestOptionsMaxCompletionTokens() { string vectorJson = """ { @@ -8357,7 +9361,7 @@ public async Task Vector117WireConformanceToRequestOptionsMaxCompletionTokens() } [Fact] - public async Task Vector118WireConformanceToRequestOptionsStopSequences() + public async Task Vector149WireConformanceToRequestOptionsStopSequences() { string vectorJson = """ { @@ -8417,7 +9421,7 @@ public async Task Vector118WireConformanceToRequestOptionsStopSequences() } [Fact] - public async Task Vector119WireConformanceToRequestResponsesSimple() + public async Task Vector150WireConformanceToRequestResponsesSimple() { string vectorJson = """ { @@ -8478,7 +9482,7 @@ public async Task Vector119WireConformanceToRequestResponsesSimple() } [Fact] - public async Task Vector120WireConformanceToRequestResponsesStructuredOutput() + public async Task Vector151WireConformanceToRequestResponsesStructuredOutput() { string vectorJson = """ { @@ -8567,7 +9571,7 @@ public async Task Vector120WireConformanceToRequestResponsesStructuredOutput() } [Fact] - public async Task Vector121WireConformanceToRequestResponsesWithTools() + public async Task Vector152WireConformanceToRequestResponsesWithTools() { string vectorJson = """ { @@ -8649,7 +9653,7 @@ public async Task Vector121WireConformanceToRequestResponsesWithTools() } [Fact] - public async Task Vector122WireConformanceToRequestStructuredOutput() + public async Task Vector153WireConformanceToRequestStructuredOutput() { string vectorJson = """ { @@ -8734,7 +9738,7 @@ public async Task Vector122WireConformanceToRequestStructuredOutput() } [Fact] - public async Task Vector123WireConformanceToRequestStructuredOutputNestedOptional() + public async Task Vector154WireConformanceToRequestStructuredOutputNestedOptional() { string vectorJson = """ { @@ -8838,7 +9842,7 @@ public async Task Vector123WireConformanceToRequestStructuredOutputNestedOptiona } [Fact] - public async Task Vector124WireConformanceToRequestToolsBindingsStripped() + public async Task Vector155WireConformanceToRequestToolsBindingsStripped() { string vectorJson = """ { @@ -8932,7 +9936,7 @@ public async Task Vector124WireConformanceToRequestToolsBindingsStripped() } [Fact] - public async Task Vector125WireConformanceToRequestToolsFunctionWire() + public async Task Vector156WireConformanceToRequestToolsFunctionWire() { string vectorJson = """ { @@ -9016,7 +10020,7 @@ public async Task Vector125WireConformanceToRequestToolsFunctionWire() } [Fact] - public async Task Vector126WireConformanceToRequestToolsNullWhenEmpty() + public async Task Vector157WireConformanceToRequestToolsNullWhenEmpty() { string vectorJson = """ { @@ -9067,7 +10071,7 @@ public async Task Vector126WireConformanceToRequestToolsNullWhenEmpty() } [Fact] - public async Task Vector127WireConformanceToRequestToolsStrictMode() + public async Task Vector158WireConformanceToRequestToolsStrictMode() { string vectorJson = """ { diff --git a/runtime/csharp/Prompty.Core/EndpointTrust.cs b/runtime/csharp/Prompty.Core/EndpointTrust.cs new file mode 100644 index 000000000..afa501c4b --- /dev/null +++ b/runtime/csharp/Prompty.Core/EndpointTrust.cs @@ -0,0 +1,263 @@ +// Copyright (c) Microsoft. All rights reserved. +using System.Collections; +using System.Globalization; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Text.RegularExpressions; + +namespace Prompty.Core; + +/// +/// Endpoint trust — keeps ambient credentials on host-approved destinations (spec §2.5.1). +/// +/// A .prompty file is untrusted input. When its connection carries no file-supplied +/// credential, executors fall back to ambient credentials (DefaultAzureCredential, +/// *_API_KEY env vars). This class decides whether the file-chosen destination may +/// receive them. +/// +public static partial class EndpointTrust +{ + /// Env var holding comma-separated, host-approved trusted origins. + public const string TrustedEndpointsEnv = "PROMPTY_TRUSTED_ENDPOINTS"; + + /// First-party provider origins that are always trusted. + public static readonly IReadOnlyList BuiltinTrustedEndpoints = + [ + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com", + ]; + + private static readonly string[] DestinationKeys = ["endpoint", "tokenUrl"]; + + /// + /// Return true if 's origin matches a built-in, host-supplied, + /// or PROMPTY_TRUSTED_ENDPOINTS trusted origin. + /// + public static bool IsTrustedEndpoint(string url, IEnumerable? trustedEndpoints = null) + { + var entries = TrustedEntries(trustedEndpoints); + if (entries.Contains("*")) + return true; + + var target = Origin(url); + if (target is null) + return false; + var (scheme, host, port) = target.Value; + + foreach (var entry in entries) + { + var allowed = Origin(entry); + if (allowed is null || allowed.Value.Scheme != scheme || allowed.Value.Port != port) + continue; + var pattern = allowed.Value.Host; + if (pattern.StartsWith("*.", StringComparison.Ordinal)) + { + if (host.EndsWith(pattern[1..], StringComparison.Ordinal)) + return true; + } + else if (pattern == host) + { + return true; + } + } + + return false; + } + + /// + /// Throw an untrusted_endpoint if ambient + /// credentials could reach a file-chosen, untrusted destination. + /// + /// The RAW (unresolved) model.connection mapping. + /// Host-supplied trusted origins. + public static void CheckConnectionTrust(object? connection, IEnumerable? trustedEndpoints = null) + { + if (connection is not IDictionary conn || Get(conn, "kind") is "reference") + return; + if (FileSuppliedCredential(conn)) + return; + + foreach (var key in DestinationKeys) + { + var destination = FileControlledDestination(Get(conn, key)); + if (destination is not null && !IsTrustedEndpoint(destination, trustedEndpoints)) + { + throw new PromptyLoadException( + "untrusted_endpoint", + $"Connection {key} '{destination}' is not trusted to receive ambient credentials. " + + $"Set the {key} via ${{env:VAR}}, add its origin to TrustedEndpoints or " + + $"{TrustedEndpointsEnv}, or use 'kind: reference'."); + } + } + } + + private static List TrustedEntries(IEnumerable? trustedEndpoints) + { + var entries = new List(BuiltinTrustedEndpoints); + if (trustedEndpoints is not null) + entries.AddRange(trustedEndpoints); + entries.AddRange((Environment.GetEnvironmentVariable(TrustedEndpointsEnv) ?? string.Empty).Split(',')); + return entries + .Where(e => e is not null && e.Trim().Length > 0) + .Select(e => e!.Trim()) + .ToList(); + } + + private static object? Get(IDictionary dict, string key) => dict.Contains(key) ? dict[key] : null; + + private static bool IsReference(string value) => + value.StartsWith("${", StringComparison.Ordinal) && value.EndsWith('}'); + + private static bool FileSuppliedCredential(IDictionary connection) + { + // Only the credential field the declared kind actually consumes counts; any other kind is ambient. + var key = Get(connection, "kind") switch + { + "key" => "apiKey", + _ => null, + }; + if (key is null) + return false; + // Blank keys count as absent: executors that trim fall back to ambient credentials. + return Get(connection, key) is string value && !string.IsNullOrWhiteSpace(value) && !IsReference(value); + } + + /// Return the effective destination if the file chose it, else null. + private static string? FileControlledDestination(object? value) + { + if (value is null) + return null; + if (value is not string str) + return Convert.ToString(value, CultureInfo.InvariantCulture); + if (IsReference(str)) + { + var inner = str[2..^1]; + var colon = inner.IndexOf(':'); + var protocol = colon < 0 ? inner : inner[..colon]; + var rest = colon < 0 ? string.Empty : inner[(colon + 1)..]; + if (protocol.Equals("env", StringComparison.OrdinalIgnoreCase)) + { + var varColon = rest.IndexOf(':'); + var varName = varColon < 0 ? rest : rest[..varColon]; + var defaultValue = varColon < 0 ? string.Empty : rest[(varColon + 1)..]; + if (Environment.GetEnvironmentVariable(varName) is not null) + return null; + return defaultValue.Length > 0 ? defaultValue : null; + } + } + + return str; + } + + // ----------------------------------------------------------------------- + // Origin parsing — mirrors Python's urllib.parse.urlsplit semantics so every + // runtime classifies the same URL identically (System.Uri differs on IDN, + // userinfo, and malformed ports). + // ----------------------------------------------------------------------- + + private readonly record struct OriginParts(string Scheme, string Host, int Port); + + [GeneratedRegex(@"^v[a-fA-F0-9]+\.[a-zA-Z0-9\._~\-!\$&'\(\)\*\+,;=:]+$")] + private static partial Regex IpvFutureRegex(); + + private static OriginParts? Origin(string url) + { + // HTTP clients treat '\' as '/', so parsers disagree on the host; reject outright. + if (url.Contains('\\')) + return null; + var s = new string(url.Trim().Where(c => c is not ('\t' or '\r' or '\n')).ToArray()); + + var scheme = string.Empty; + var rest = s; + var colon = s.IndexOf(':'); + if (colon > 0 && char.IsAsciiLetter(s[0]) + && s[..colon].All(c => char.IsAsciiLetterOrDigit(c) || c is '+' or '-' or '.')) + { + scheme = s[..colon].ToLowerInvariant(); + rest = s[(colon + 1)..]; + } + + if (scheme is not ("http" or "https")) + return null; + + var netloc = string.Empty; + if (rest.StartsWith("//", StringComparison.Ordinal)) + { + var after = rest[2..]; + var end = after.IndexOfAny(['/', '?', '#']); + netloc = end < 0 ? after : after[..end]; + } + + if (netloc.Contains('[') != netloc.Contains(']')) + return null; + if (!CheckNetloc(netloc)) + return null; + + var at = netloc.LastIndexOf('@'); + var hostinfo = at < 0 ? netloc : netloc[(at + 1)..]; + string hostname; + string port; + var openBracket = hostinfo.IndexOf('['); + if (openBracket >= 0) + { + var bracketed = hostinfo[(openBracket + 1)..]; + var close = bracketed.IndexOf(']'); + hostname = close < 0 ? bracketed : bracketed[..close]; + var afterClose = close < 0 ? string.Empty : bracketed[(close + 1)..]; + var portColon = afterClose.IndexOf(':'); + port = portColon < 0 ? string.Empty : afterClose[(portColon + 1)..]; + if (!IsValidBracketedHost(hostname)) + return null; + } + else + { + var portColon = hostinfo.IndexOf(':'); + hostname = portColon < 0 ? hostinfo : hostinfo[..portColon]; + port = portColon < 0 ? string.Empty : hostinfo[(portColon + 1)..]; + } + + hostname = hostname.ToLowerInvariant(); + if (hostname.Length == 0) + return null; + + int portNumber; + if (port.Length == 0) + { + portNumber = scheme == "http" ? 80 : 443; + } + else + { + if (!port.All(char.IsAsciiDigit)) + return null; + var trimmed = port.TrimStart('0'); + if (trimmed.Length > 5 || !int.TryParse(trimmed.Length == 0 ? "0" : trimmed, out portNumber) + || portNumber > 65535) + return null; + } + + return new OriginParts(scheme, hostname, portNumber); + } + + private static bool CheckNetloc(string netloc) + { + if (netloc.Length == 0 || netloc.All(char.IsAscii)) + return true; + var n = netloc.Replace("@", "").Replace(":", "").Replace("#", "").Replace("?", ""); + var normalized = n.Normalize(NormalizationForm.FormKC); + if (n == normalized) + return true; + return normalized.IndexOfAny(['/', '?', '#', '@', ':']) < 0; + } + + private static bool IsValidBracketedHost(string host) + { + if (host.StartsWith('v')) + return IpvFutureRegex().IsMatch(host); + return IPAddress.TryParse(host, out var ip) && ip.AddressFamily == AddressFamily.InterNetworkV6; + } +} diff --git a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs index 57b39cb03..381e6cf18 100644 --- a/runtime/csharp/Prompty.Core/Model/VectorConformance.cs +++ b/runtime/csharp/Prompty.Core/Model/VectorConformance.cs @@ -387,6 +387,384 @@ public static async Task RunLoadConformanceConformanceAsync(ILoadConformance sea if (Canonical(actual.ToJson()) != Canonical("{\"tools\":[{\"name\":\"weather_api\",\"kind\":\"openapi\",\"specification\":\"./weather.json\"}]}")) throw new SeamConformanceException("tools_openapi_load misrouted"); } + { + // vector: trusted_endpoint_builtin_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}")) + throw new SeamConformanceException("trusted_endpoint_builtin_allowed misrouted"); + } + { + // vector: trusted_endpoint_default_port_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}")) + throw new SeamConformanceException("trusted_endpoint_default_port_allowed misrouted"); + } + { + // vector: trusted_endpoint_env_endpoint_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}}")) + throw new SeamConformanceException("trusted_endpoint_env_endpoint_allowed misrouted"); + } + { + // vector: trusted_endpoint_env_list_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}}")) + throw new SeamConformanceException("trusted_endpoint_env_list_allowed misrouted"); + } + { + // vector: trusted_endpoint_literal_key_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}}")) + throw new SeamConformanceException("trusted_endpoint_literal_key_allowed misrouted"); + } + { + // vector: trusted_endpoint_option_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}}")) + throw new SeamConformanceException("trusted_endpoint_option_allowed misrouted"); + } + { + // vector: trusted_endpoint_reference_skipped + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}")) + throw new SeamConformanceException("trusted_endpoint_reference_skipped misrouted"); + } + { + // vector: trusted_endpoint_userinfo_allowed + var source = JsonSerializer.Deserialize("null")!; + var actual = await seam.LoadAsync(source); + if (Canonical(actual.ToJson()) != Canonical("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}")) + throw new SeamConformanceException("trusted_endpoint_userinfo_allowed misrouted"); + } + { + // vector: untrusted_endpoint_ambient_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_ambient_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_backslash_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_backslash_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_blank_literal_key_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_blank_literal_key_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_env_default_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_env_default_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_env_key_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_env_key_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_file_connection_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_file_connection_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_fragment_before_at_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_fragment_before_at_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_ipv6_literal_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_ipv6_literal_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_key_with_client_secret_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_key_with_client_secret_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_mismatched_credential_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_mismatched_credential_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_no_key_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_no_key_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_oauth_literal_secret_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_oauth_literal_secret_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_percent_encoded_host_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_percent_encoded_host_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_port_mismatch_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_port_mismatch_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_query_before_at_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_query_before_at_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_scheme_mismatch_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_scheme_mismatch_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_suffix_lookalike_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_suffix_lookalike_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_suffix_spoof_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_suffix_spoof_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_trailing_dot_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_trailing_dot_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_unicode_homoglyph_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_unicode_homoglyph_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_userinfo_lookalike_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_userinfo_lookalike_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_wildcard_apex_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_wildcard_apex_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_wildcard_boundary_rejected + var source = JsonSerializer.Deserialize("null")!; + Exception? caught = null; + try + { + await seam.LoadAsync(source); + } + catch (Exception error) + { + caught = error; + } + if (caught is null) throw new SeamConformanceException("untrusted_endpoint_wildcard_boundary_rejected: expected an error"); + } } } diff --git a/runtime/csharp/Prompty.Core/PromptyLoadException.cs b/runtime/csharp/Prompty.Core/PromptyLoadException.cs index 23b0561f9..4192e584b 100644 --- a/runtime/csharp/Prompty.Core/PromptyLoadException.cs +++ b/runtime/csharp/Prompty.Core/PromptyLoadException.cs @@ -7,7 +7,7 @@ namespace Prompty.Core; /// /// The kind is drawn from the cross-runtime load-error taxonomy /// (env_var_not_set, file_reference, invalid_frontmatter, -/// invalid_template, ...), letting conformance adapters map a runtime failure to +/// invalid_template, untrusted_endpoint, ...), letting conformance adapters map a runtime failure to /// the spec's expectedError by exception type — never by message /// substring matching. /// diff --git a/runtime/csharp/Prompty.Core/PromptyLoadOptions.cs b/runtime/csharp/Prompty.Core/PromptyLoadOptions.cs index 0f269fdfd..acad8eac5 100644 --- a/runtime/csharp/Prompty.Core/PromptyLoadOptions.cs +++ b/runtime/csharp/Prompty.Core/PromptyLoadOptions.cs @@ -12,4 +12,11 @@ public sealed class PromptyLoadOptions /// The prompt file's directory is always allowed. /// public IEnumerable AllowedFileRoots { get; init; } = []; + + /// + /// Additional origins (e.g. http://localhost:11434, https://*.contoso.com) + /// trusted to receive ambient credentials (spec §2.5.1). Unioned with the built-in + /// provider origins and the PROMPTY_TRUSTED_ENDPOINTS env var; "*" trusts all. + /// + public IEnumerable TrustedEndpoints { get; init; } = []; } diff --git a/runtime/csharp/Prompty.Core/PromptyLoader.cs b/runtime/csharp/Prompty.Core/PromptyLoader.cs index 74732ddbc..973e2747a 100644 --- a/runtime/csharp/Prompty.Core/PromptyLoader.cs +++ b/runtime/csharp/Prompty.Core/PromptyLoader.cs @@ -4,7 +4,7 @@ namespace Prompty.Core; /// /// Loads .prompty files into typed Prompty instances. -/// Flow: read file → parse frontmatter/body → resolve ${env:}/${file:} refs → Prompty.Load(). +/// Flow: read file → parse frontmatter/body → check endpoint trust (§2.5.1) → resolve ${env:}/${file:} refs → Prompty.Load(). /// public static class PromptyLoader { @@ -91,7 +91,12 @@ private static Agent Build(string contents, string fullPath, PromptyLoadOptions? $"Template must be an object with format/parser, got string '{templateString}'."); } - // 2. Load via typed model with ${env:}/${file:} resolution + // 2. Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. + EndpointTrust.CheckConnectionTrust( + ReferenceResolver.RawConnection(data, fullPath, options?.AllowedFileRoots), + options?.TrustedEndpoints); + + // 3. Load via typed model with ${env:}/${file:} resolution var ctx = new LoadContext { PreProcess = ReferenceResolver.CreatePreProcess(fullPath, options?.AllowedFileRoots), @@ -99,7 +104,7 @@ private static Agent Build(string contents, string fullPath, PromptyLoadOptions? var agent = Agent.Load(data, ctx); - // 3. Attach source path in metadata + // 4. Attach source path in metadata agent.Metadata ??= new Dictionary(); agent.Metadata["__source_path"] = fullPath; diff --git a/runtime/csharp/Prompty.Core/README.md b/runtime/csharp/Prompty.Core/README.md index a8f4f2b23..81ce2a6c1 100644 --- a/runtime/csharp/Prompty.Core/README.md +++ b/runtime/csharp/Prompty.Core/README.md @@ -101,6 +101,29 @@ var agent = PromptyLoader.Load("chat.prompty"); var agent = await PromptyLoader.LoadAsync("chat.prompty"); ``` +#### Load options + +```csharp +var agent = PromptyLoader.Load("chat.prompty", new PromptyLoadOptions +{ + // Extra directories ${file:...} references may read from (the prompt's directory is always allowed). + AllowedFileRoots = ["/shared/config"], + // Extra origins trusted to receive ambient credentials (spec §2.5.1). + TrustedEndpoints = ["http://localhost:11434", "https://*.contoso.com"], +}); +``` + +**Endpoint trust.** A `.prompty` file is untrusted input, so the loader refuses +(`PromptyLoadException` with `Kind == "untrusted_endpoint"`) to load a file whose +`model.connection` would send ambient credentials (`DefaultAzureCredential` tokens, +`*_API_KEY` env vars) to an endpoint or `tokenUrl` the file chose. Built-in provider +origins (`https://api.openai.com`, `https://api.anthropic.com`, `https://*.openai.azure.com`, +`https://*.services.ai.azure.com`, `https://*.cognitiveservices.azure.com`, +`https://login.microsoftonline.com`) are always trusted. To allow another destination, +set the endpoint via `${env:VAR}`, add its origin to `TrustedEndpoints` or the +comma-separated `PROMPTY_TRUSTED_ENDPOINTS` env var (`*` trusts all), supply a literal +credential in the file, or use `kind: reference`. + ### Pipeline ```csharp diff --git a/runtime/csharp/Prompty.Core/ReferenceResolver.cs b/runtime/csharp/Prompty.Core/ReferenceResolver.cs index dd8a71ecf..d4519fdf2 100644 --- a/runtime/csharp/Prompty.Core/ReferenceResolver.cs +++ b/runtime/csharp/Prompty.Core/ReferenceResolver.cs @@ -33,6 +33,66 @@ public static class ReferenceResolver return data => ResolveReferences(data, parentDir, allowedRoots); } + /// + /// Return the unresolved model.connection from raw frontmatter, following + /// whole-value ${file:...} references for model and model.connection + /// with the same containment rules as reference resolution (spec §2.5.1). + /// + internal static object? RawConnection( + Dictionary data, + string promptyFilePath, + IEnumerable? allowedFileRoots = null) + { + var parentDir = Path.GetDirectoryName(GetCanonicalPath(promptyFilePath)) + ?? throw new ArgumentException($"Cannot determine parent directory of '{promptyFilePath}'"); + var allowedRoots = new[] { parentDir } + .Concat(allowedFileRoots ?? []) + .Select(GetCanonicalPath) + .ToArray(); + + object? Follow(object? value, string key) + { + if (value is string str && str.StartsWith(RefPrefix) && str.EndsWith(RefSuffix)) + { + var inner = str[RefPrefix.Length..^RefSuffix.Length]; + var colonIndex = inner.IndexOf(':'); + if (colonIndex >= 0 && inner[..colonIndex].Equals("file", StringComparison.OrdinalIgnoreCase)) + return LoadContainedFile(inner[(colonIndex + 1)..], parentDir, allowedRoots, key); + } + return value; + } + + var model = Follow(data.GetValueOrDefault("model"), "model"); + if (model is not System.Collections.IDictionary modelDict) + return null; + return Follow(modelDict.Contains("connection") ? modelDict["connection"] : null, "connection"); + } + + /// + /// Containment-check then load a ${file:} target; returns null if it does not exist. + /// + private static object? LoadContainedFile( + string relativePath, + string parentDir, + IReadOnlyCollection allowedRoots, + string key) + { + var fullPath = GetCanonicalPath(Path.IsPathRooted(relativePath) + ? Path.GetFullPath(relativePath) + : Path.GetFullPath(Path.Combine(parentDir, relativePath))); + if (!allowedRoots.Any(root => IsWithinRoot(fullPath, root))) + { + throw new PromptyLoadException( + "file_reference", + $"File reference '{relativePath}' resolves outside allowed roots (resolved to '{fullPath}', key: '{key}')."); + } + + if (!File.Exists(fullPath)) + return null; + + return LoadFileContent(File.ReadAllText(fullPath), fullPath); + } + /// /// Walks a dictionary and resolves any string values matching ${protocol:value} patterns. /// Only processes top-level string values in the given dictionary (recursive walking is @@ -132,7 +192,11 @@ private static object ResolveFileRef( $"File reference '{relativePath}' resolves outside allowed roots (resolved to '{fullPath}', key: '{key}')."); } - var content = File.ReadAllText(fullPath); + return LoadFileContent(File.ReadAllText(fullPath), fullPath); + } + + private static object LoadFileContent(string content, string fullPath) + { var ext = Path.GetExtension(fullPath).ToLowerInvariant(); return ext switch diff --git a/runtime/go/prompty/model/endpoint_trust.go b/runtime/go/prompty/model/endpoint_trust.go new file mode 100644 index 000000000..e53736902 --- /dev/null +++ b/runtime/go/prompty/model/endpoint_trust.go @@ -0,0 +1,218 @@ +package prompty + +// Endpoint trust — keep ambient credentials on host-approved destinations +// (spec §2.5.1). +// +// A .prompty file is untrusted input. When its connection carries no +// file-supplied credential, executors fall back to ambient credentials +// (DefaultAzureCredential, *_API_KEY env vars). This file decides whether the +// file-chosen destination may receive them. + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" +) + +// TrustedEndpointsEnv names the env var holding extra comma-separated trusted origins. +const TrustedEndpointsEnv = "PROMPTY_TRUSTED_ENDPOINTS" + +// BuiltinTrustedEndpoints are first-party provider origins trusted without host configuration. +var BuiltinTrustedEndpoints = []string{ + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com", +} + +var defaultPorts = map[string]int{"http": 80, "https": 443} + +// credentialKeys maps a connection kind to the credential field it consumes. +var credentialKeys = map[string]string{"key": "apiKey"} + +var destinationKeys = []string{"endpoint", "tokenUrl"} + +type origin struct { + scheme string + host string + port int +} + +// parseOrigin returns the (scheme, host, port) of an absolute http(s) URL. +func parseOrigin(raw string) (origin, bool) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return origin{}, false + } + scheme := strings.ToLower(u.Scheme) + host := strings.ToLower(u.Hostname()) + def, ok := defaultPorts[scheme] + if !ok || host == "" { + return origin{}, false + } + port := def + if p := u.Port(); p != "" { + n, err := strconv.Atoi(p) + if err != nil || n < 0 || n > 65535 { + return origin{}, false + } + port = n + } + return origin{scheme: scheme, host: host, port: port}, true +} + +func trustedEntries(trustedEndpoints []string) []string { + entries := make([]string, 0, len(BuiltinTrustedEndpoints)+len(trustedEndpoints)) + entries = append(entries, BuiltinTrustedEndpoints...) + entries = append(entries, trustedEndpoints...) + entries = append(entries, strings.Split(os.Getenv(TrustedEndpointsEnv), ",")...) + out := entries[:0] + for _, e := range entries { + if t := strings.TrimSpace(e); t != "" { + out = append(out, t) + } + } + return out +} + +// IsTrustedEndpoint reports whether rawURL's origin matches a built-in, host +// option, or PROMPTY_TRUSTED_ENDPOINTS origin. +func IsTrustedEndpoint(rawURL string, trustedEndpoints []string) bool { + entries := trustedEntries(trustedEndpoints) + for _, e := range entries { + if e == "*" { + return true + } + } + target, ok := parseOrigin(rawURL) + if !ok { + return false + } + for _, entry := range entries { + allowed, ok := parseOrigin(entry) + if !ok || allowed.scheme != target.scheme || allowed.port != target.port { + continue + } + if strings.HasPrefix(allowed.host, "*.") { + if strings.HasSuffix(target.host, allowed.host[1:]) { + return true + } + } else if allowed.host == target.host { + return true + } + } + return false +} + +func isReference(value string) bool { + return strings.HasPrefix(value, "${") && strings.HasSuffix(value, "}") +} + +func fileSuppliedCredential(connection map[string]interface{}) bool { + // Only the credential field the declared kind actually consumes counts; any other kind is ambient. + kind, _ := connection["kind"].(string) + key, ok := credentialKeys[kind] + if !ok { + return false + } + value, ok := connection[key].(string) + // Blank keys count as absent: executors that trim fall back to ambient credentials. + return ok && strings.TrimSpace(value) != "" && !isReference(value) +} + +// fileControlledDestination returns the effective destination if the file chose it. +func fileControlledDestination(value interface{}) (string, bool) { + if value == nil { + return "", false + } + str, ok := value.(string) + if !ok { + return fmt.Sprintf("%v", value), true + } + if isReference(str) { + protocol, rest, _ := strings.Cut(str[2:len(str)-1], ":") + if strings.ToLower(protocol) == "env" { + varName, def, _ := strings.Cut(rest, ":") + if _, set := os.LookupEnv(varName); set { + return "", false + } + return def, def != "" + } + } + return str, true +} + +// CheckConnectionTrust returns an untrusted_endpoint *LoadError if ambient +// credentials could reach a file-chosen, untrusted destination. connection is +// the RAW (unresolved) model.connection value. +func CheckConnectionTrust(connection interface{}, trustedEndpoints []string) error { + conn, ok := connection.(map[string]interface{}) + if !ok || conn["kind"] == "reference" { + return nil + } + if fileSuppliedCredential(conn) { + return nil + } + for _, key := range destinationKeys { + destination, controlled := fileControlledDestination(conn[key]) + if controlled && !IsTrustedEndpoint(destination, trustedEndpoints) { + return &LoadError{ + Kind: "untrusted_endpoint", + Message: fmt.Sprintf( + "Connection %s '%s' is not trusted to receive ambient credentials. "+ + "Set the %s via ${env:VAR}, add its origin to TrustedEndpoints or %s, "+ + "or use 'kind: reference'.", + key, destination, key, TrustedEndpointsEnv), + } + } + } + return nil +} + +// CheckEndpointTrust applies spec §2.5.1 to RAW (unresolved) frontmatter. It +// follows whole-value ${file:} references for `model` and `model.connection` +// with the same containment as reference resolution, so it must run before +// ResolveReferences. +func CheckEndpointTrust(data map[string]interface{}, parentDir string, allowedRoots, trustedEndpoints []string) error { + conn, err := rawConnection(data, parentDir, allowedRoots) + if err != nil { + return err + } + return CheckConnectionTrust(conn, trustedEndpoints) +} + +func rawConnection(data map[string]interface{}, parentDir string, allowedRoots []string) (interface{}, error) { + follow := func(value interface{}) (interface{}, error) { + str, ok := value.(string) + if !ok || !isReference(str) { + return value, nil + } + protocol, ref, _ := strings.Cut(str[2:len(str)-1], ":") + if strings.ToLower(protocol) != "file" { + return value, nil + } + full := ref + if !filepath.IsAbs(full) { + full = filepath.Join(parentDir, ref) + } + // A missing file is reported by reference resolution, not here. + if _, err := os.Stat(full); err != nil { + return nil, nil + } + return resolveFile(ref, parentDir, allowedRoots) + } + model, err := follow(data["model"]) + if err != nil { + return nil, err + } + modelMap, ok := model.(map[string]interface{}) + if !ok { + return nil, nil + } + return follow(modelMap["connection"]) +} diff --git a/runtime/go/prompty/model/endpoint_trust_test.go b/runtime/go/prompty/model/endpoint_trust_test.go new file mode 100644 index 000000000..4b8a46302 --- /dev/null +++ b/runtime/go/prompty/model/endpoint_trust_test.go @@ -0,0 +1,233 @@ +package prompty + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeTrustPrompty(t *testing.T, dir, connection string) string { + t.Helper() + p := filepath.Join(dir, "t.prompty") + content := "---\nname: t\nmodel:\n id: gpt-4\n connection:\n" + connection + "\n---\nHello\n" + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func expectUntrusted(t *testing.T, err error, contains string) { + t.Helper() + var le *LoadError + if !errors.As(err, &le) || le.Kind != "untrusted_endpoint" { + t.Fatalf("expected untrusted_endpoint LoadError, got %v", err) + } + if contains != "" && !strings.Contains(le.Message, contains) { + t.Fatalf("expected message to contain %q, got %q", contains, le.Message) + } +} + +func connectionEndpoint(t *testing.T, agent Agent) string { + t.Helper() + saved := agent.Save(NewSaveContext()) + model, _ := saved["model"].(map[string]interface{}) + conn, _ := model["connection"].(map[string]interface{}) + endpoint, _ := conn["endpoint"].(string) + return endpoint +} + +func TestIsTrustedEndpointBuiltins(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + for _, u := range []string{ + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + } { + if !IsTrustedEndpoint(u, nil) { + t.Errorf("expected %q to be trusted", u) + } + } +} + +func TestIsTrustedEndpointUntrusted(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + for _, u := range []string{ + "http://api.openai.com", // scheme mismatch + "https://api.openai.com:8443", // port mismatch + "https://openai.azure.com", // wildcard never matches apex + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", // userinfo ignored; host is evil.com + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "", + } { + if IsTrustedEndpoint(u, nil) { + t.Errorf("expected %q to be untrusted", u) + } + } +} + +func TestIsTrustedEndpointHostOptionAndWildcard(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + if !IsTrustedEndpoint("http://localhost:11434/v1", []string{"http://LOCALHOST:11434"}) { + t.Error("expected host option to trust localhost") + } + if !IsTrustedEndpoint("https://x.contoso.com", []string{"https://*.contoso.com"}) { + t.Error("expected wildcard to trust subdomain") + } + if IsTrustedEndpoint("https://contoso.com", []string{"https://*.contoso.com"}) { + t.Error("wildcard must not match apex") + } +} + +func TestIsTrustedEndpointEnvListAndStar(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, " https://a.example.com , http://localhost:8080") + if !IsTrustedEndpoint("http://localhost:8080/x", nil) { + t.Error("expected env list to trust localhost:8080") + } + if IsTrustedEndpoint("http://localhost:8081", nil) { + t.Error("expected localhost:8081 to be untrusted") + } + t.Setenv(TrustedEndpointsEnv, "*") + if !IsTrustedEndpoint("http://anything.example", nil) { + t.Error("expected * to trust everything") + } +} + +func TestLoadAmbientLiteralEndpointRejected(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + p := writeTrustPrompty(t, t.TempDir(), " kind: foundry\n endpoint: http://127.0.0.1:9999") + _, err := LoadPromptyFile(p) + expectUntrusted(t, err, "127.0.0.1") +} + +func TestLoadEnvKeyLiteralEndpointRejected(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + t.Setenv("SOME_KEY", "secret") + p := writeTrustPrompty(t, t.TempDir(), " kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}") + _, err := LoadPromptyFile(p) + expectUntrusted(t, err, "not trusted") +} + +func TestLoadOAuthTokenURLChecked(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + t.Setenv("SECRET", "s") + conn := " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token" + _, err := LoadPromptyFile(writeTrustPrompty(t, t.TempDir(), conn)) + expectUntrusted(t, err, "tokenUrl") +} + +func TestLoadMismatchedCredentialFieldRejected(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + dir := t.TempDir() + _, err := LoadPromptyFile(writeTrustPrompty(t, dir, " kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk")) + expectUntrusted(t, err, "not trusted") + _, err = LoadPromptyFile(writeTrustPrompty(t, dir, " kind: key\n endpoint: https://evil.example.com\n clientSecret: junk")) + expectUntrusted(t, err, "not trusted") +} + +func TestLoadLiteralKeyAllowed(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + agent, err := LoadPromptyFile(writeTrustPrompty(t, t.TempDir(), " kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine")) + if err != nil { + t.Fatal(err) + } + if got := connectionEndpoint(t, agent); got != "http://127.0.0.1:9999" { + t.Fatalf("endpoint = %q", got) + } +} + +func TestLoadEnvEndpointAllowed(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + t.Setenv("EP", "http://127.0.0.1:9999") + agent, err := LoadPromptyFile(writeTrustPrompty(t, t.TempDir(), " kind: foundry\n endpoint: ${env:EP}")) + if err != nil { + t.Fatal(err) + } + if got := connectionEndpoint(t, agent); got != "http://127.0.0.1:9999" { + t.Fatalf("endpoint = %q", got) + } +} + +func TestLoadEnvDefaultUsedRejected(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + os.Unsetenv("UNSET_EP") + _, err := LoadPromptyFile(writeTrustPrompty(t, t.TempDir(), " kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}")) + expectUntrusted(t, err, "not trusted") +} + +func TestLoadTrustedOptionAllows(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + p := writeTrustPrompty(t, t.TempDir(), " kind: foundry\n endpoint: http://127.0.0.1:9999/api") + if _, err := LoadPromptyFile(p); err == nil { + t.Fatal("expected rejection without option") + } + agent, err := LoadPromptyFileWithOptions(p, LoadOptions{TrustedEndpoints: []string{"http://127.0.0.1:9999"}}) + if err != nil { + t.Fatal(err) + } + if got := connectionEndpoint(t, agent); got != "http://127.0.0.1:9999/api" { + t.Fatalf("endpoint = %q", got) + } +} + +func TestLoadFileModelReferenceChecked(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + dir := t.TempDir() + model, _ := json.Marshal(map[string]interface{}{ + "id": "gpt-4", + "connection": map[string]interface{}{"kind": "foundry", "endpoint": "https://evil.example.com"}, + }) + if err := os.WriteFile(filepath.Join(dir, "model.json"), model, 0o644); err != nil { + t.Fatal(err) + } + p := filepath.Join(dir, "t.prompty") + if err := os.WriteFile(p, []byte("---\nname: t\nmodel: ${file:model.json}\n---\nHi\n"), 0o644); err != nil { + t.Fatal(err) + } + _, err := LoadPromptyFile(p) + expectUntrusted(t, err, "not trusted") +} + +func TestLoadFileTraversalStillFileReference(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + base := t.TempDir() + app := filepath.Join(base, "app") + if err := os.MkdirAll(app, 0o755); err != nil { + t.Fatal(err) + } + secret, _ := json.Marshal(map[string]interface{}{"kind": "foundry", "endpoint": "https://evil.example.com"}) + if err := os.WriteFile(filepath.Join(base, "secret.json"), secret, 0o644); err != nil { + t.Fatal(err) + } + p := filepath.Join(app, "t.prompty") + content := "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:../secret.json}\n---\nHi\n" + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + _, err := LoadPromptyFile(p) + var le *LoadError + if !errors.As(err, &le) || le.Kind != "file_traversal" { + t.Fatalf("expected file_traversal, got %v", err) + } +} + +func TestLoadFrontmatterCannotSelfTrust(t *testing.T) { + t.Setenv(TrustedEndpointsEnv, "") + p := filepath.Join(t.TempDir(), "t.prompty") + content := "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n" + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + _, err := LoadPromptyFile(p) + expectUntrusted(t, err, "") +} diff --git a/runtime/go/prompty/model/loader.go b/runtime/go/prompty/model/loader.go index 22aa6cb29..efa54802a 100644 --- a/runtime/go/prompty/model/loader.go +++ b/runtime/go/prompty/model/loader.go @@ -22,7 +22,7 @@ var frontmatterRe = regexp.MustCompile(`(?s)^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+ // LoadError classifies a load failure so the conformance layer (and callers) // can map it onto the expected error contract. type LoadError struct { - Kind string // env | file_missing | file_traversal | template | frontmatter | not_found | required_input + Kind string // env | file_missing | file_traversal | template | frontmatter | not_found | required_input | untrusted_endpoint Field string Message string } @@ -219,22 +219,59 @@ func BuildAgentFromData(data map[string]interface{}) (Agent, error) { return LoadAgent(data, NewLoadContext()) } +// LoadOptions carries host load options (spec §4.1). Frontmatter can never set these. +type LoadOptions struct { + // AllowedFileRoots are extra directories ${file:} references may read from, + // in addition to the prompt file's directory (spec §2.11). + AllowedFileRoots []string + // TrustedEndpoints are extra origins allowed to receive ambient credentials + // (spec §2.5.1), e.g. "https://my-proxy.contoso.com" or "https://*.contoso.com". + TrustedEndpoints []string +} + // LoadPromptyContent parses .prompty text, resolves references relative to // parentDir, and returns the loaded Agent. func LoadPromptyContent(contents, parentDir string, allowedRoots []string) (Agent, error) { + return loadPromptyContent(contents, parentDir, allowedRoots, nil) +} + +// LoadPromptyContentWithOptions is LoadPromptyContent with host load options. +// opts.AllowedFileRoots are appended to allowedRoots. +func LoadPromptyContentWithOptions(contents, parentDir string, allowedRoots []string, opts LoadOptions) (Agent, error) { + return loadPromptyContent(contents, parentDir, appendRoots(allowedRoots, opts.AllowedFileRoots), opts.TrustedEndpoints) +} + +func loadPromptyContent(contents, parentDir string, allowedRoots, trustedEndpoints []string) (Agent, error) { data, err := ParseFrontmatter(contents) if err != nil { return Agent{}, err } + // Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. + if err := CheckEndpointTrust(data, parentDir, allowedRoots, trustedEndpoints); err != nil { + return Agent{}, err + } if err := ResolveReferences(data, parentDir, allowedRoots); err != nil { return Agent{}, err } return BuildAgentFromData(data) } +func appendRoots(roots, extra []string) []string { + out := append([]string{}, roots...) + for _, r := range extra { + out = append(out, canonicalPath(r)) + } + return out +} + // LoadPromptyFile reads a .prompty file from disk and loads it, resolving // references relative to the file's directory. func LoadPromptyFile(path string) (Agent, error) { + return LoadPromptyFileWithOptions(path, LoadOptions{}) +} + +// LoadPromptyFileWithOptions is LoadPromptyFile with host load options. +func LoadPromptyFileWithOptions(path string, opts LoadOptions) (Agent, error) { if _, err := os.Stat(path); err != nil { return Agent{}, &LoadError{Kind: "not_found", Message: fmt.Sprintf("FileNotFoundError: prompty file not found: '%s'", path)} } @@ -243,7 +280,7 @@ func LoadPromptyFile(path string) (Agent, error) { return Agent{}, &LoadError{Kind: "not_found", Message: fmt.Sprintf("FileNotFoundError: %v", err)} } dir := canonicalPath(filepath.Dir(path)) - return LoadPromptyContent(string(content), dir, []string{dir}) + return LoadPromptyContentWithOptions(string(content), dir, []string{dir}, opts) } // ValidateInputs fills declared input defaults and reports missing required diff --git a/runtime/go/prompty/model/vector_conformance_test.go b/runtime/go/prompty/model/vector_conformance_test.go index 279217c44..f4444e0e4 100644 --- a/runtime/go/prompty/model/vector_conformance_test.go +++ b/runtime/go/prompty/model/vector_conformance_test.go @@ -811,7 +811,10 @@ func TestVector21LoadConformanceLoadConnectionTypesLoad(t *testing.T) { "endpoint": "https://localhost:8080" } } - } + }, + "trustedEndpoints": [ + "https://localhost:8080" + ] }, "expected": { "model": { @@ -911,7 +914,8 @@ func TestVector24LoadConformanceLoadEnvDefault(t *testing.T) { "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "${env:MISSING_VAR:fallback_value}" + "endpoint": "${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }, @@ -1726,7 +1730,976 @@ func TestVector47LoadConformanceLoadToolsOpenapiLoad(t *testing.T) { vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) } -func TestVector48MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { +func TestVector48LoadConformanceLoadTrustedEndpointBuiltinAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_builtin_allowed", + "description": "Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-builtin", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector49LoadConformanceLoadTrustedEndpointDefaultPortAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_default_port_allowed", + "description": "An explicit default port normalizes to the built-in origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-default-port-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector50LoadConformanceLoadTrustedEndpointEnvEndpointAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_env_endpoint_allowed", + "description": "An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-endpoint", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:VECTOR_FOUNDRY_ENDPOINT}" + } + } + }, + "env": { + "VECTOR_FOUNDRY_ENDPOINT": "https://host-chosen.example.com" + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://host-chosen.example.com" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector51LoadConformanceLoadTrustedEndpointEnvListAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_env_list_allowed", + "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-list", + "model": { + "id": "llama3", + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "env": { + "PROMPTY_TRUSTED_ENDPOINTS": "https://other.example.com, http://localhost:11434" + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector52LoadConformanceLoadTrustedEndpointLiteralKeyAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_literal_key_allowed", + "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-literal-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com", + "apiKey": "file-supplied-key" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector53LoadConformanceLoadTrustedEndpointOptionAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_option_allowed", + "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-option", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "trustedEndpoints": [ + "HTTPS://PROXY.Example.com" + ] + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector54LoadConformanceLoadTrustedEndpointReferenceSkipped(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_reference_skipped", + "description": "kind: reference connections carry no endpoint and are not subject to the trust check", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-reference", + "model": { + "id": "gpt-4", + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector55LoadConformanceLoadTrustedEndpointUserinfoAllowed(t *testing.T) { + vectorJSON := `{ + "name": "trusted_endpoint_userinfo_allowed", + "description": "Userinfo is ignored; the destination host api.openai.com is trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-userinfo-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector56LoadConformanceLoadUntrustedEndpointAmbientRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_ambient_rejected", + "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ambient", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com/api/projects/p" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector57LoadConformanceLoadUntrustedEndpointBackslashRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_backslash_rejected", + "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-backslash", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com\\@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector58LoadConformanceLoadUntrustedEndpointBlankLiteralKeyRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_blank_literal_key_rejected", + "description": "a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-blank-key", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": " " + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector59LoadConformanceLoadUntrustedEndpointEnvDefaultRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_env_default_rejected", + "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-default", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:NONEXISTENT:https://attacker.example.com}" + } + } + }, + "env": {} + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector60LoadConformanceLoadUntrustedEndpointEnvKeyRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_env_key_rejected", + "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": "${env:VECTOR_HOST_KEY}" + } + } + }, + "env": { + "VECTOR_HOST_KEY": "host-secret" + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector61LoadConformanceLoadUntrustedEndpointFileConnectionRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_file_connection_rejected", + "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-file-conn", + "model": { + "id": "gpt-4", + "connection": "${file:conn.json}" + } + }, + "files": { + "conn.json": { + "kind": "foundry", + "endpoint": "https://attacker.example.com" + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector62LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_fragment_before_at_rejected", + "description": "A '#' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-fragment-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com#@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector63LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_ipv6_literal_rejected", + "description": "An IPv6 literal is not a trusted origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ipv6-literal", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://[::1]:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector64LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_key_with_client_secret_rejected", + "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-key-client-secret", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "clientSecret": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector65LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_mismatched_credential_rejected", + "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-mismatched-credential", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com", + "apiKey": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector66LoadConformanceLoadUntrustedEndpointNoKeyRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_no_key_rejected", + "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-no-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "http://127.0.0.1:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector67LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_oauth_literal_secret_rejected", + "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-oauth-literal", + "model": { + "id": "gpt-4", + "provider": "anthropic", + "connection": { + "kind": "oauth", + "endpoint": "https://attacker.example.com", + "clientId": "decoy", + "clientSecret": "decoy", + "tokenUrl": "https://login.microsoftonline.com/t/oauth2/v2.0/token" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector68LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_percent_encoded_host_rejected", + "description": "Hosts are compared without percent-decoding, so an encoded host never matches", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-percent-encoded-host", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com%2eevil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector69LoadConformanceLoadUntrustedEndpointPortMismatchRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_port_mismatch_rejected", + "description": "A non-default port is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-port-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com:8443" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector70LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_query_before_at_rejected", + "description": "A '?' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-query-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com?@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector71LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_scheme_mismatch_rejected", + "description": "Built-in origins are https; http on the same host is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-scheme-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector72LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_suffix_lookalike_rejected", + "description": "A trusted host as a label prefix of another domain is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-suffix-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com.evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector73LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_suffix_spoof_rejected", + "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-spoof", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://good.example.com.attacker.example.com" + } + } + }, + "trustedEndpoints": [ + "https://good.example.com" + ] + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector74LoadConformanceLoadUntrustedEndpointTrailingDotRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_trailing_dot_rejected", + "description": "Hosts are compared exactly; a trailing-dot host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-trailing-dot", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com." + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector75LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_unicode_homoglyph_rejected", + "description": "A non-ASCII lookalike host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-unicode-homoglyph", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://аpi.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector76LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_userinfo_lookalike_rejected", + "description": "A trusted host in userinfo is not the destination host", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-userinfo-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com@evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector77LoadConformanceLoadUntrustedEndpointWildcardApexRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_wildcard_apex_rejected", + "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-apex", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://myres.openai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector78LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected(t *testing.T) { + vectorJSON := `{ + "name": "untrusted_endpoint_wildcard_boundary_rejected", + "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-wildcard-boundary", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evilservices.ai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" +}` + var vector map[string]any + if err := json.Unmarshal([]byte(vectorJSON), &vector); err != nil { + t.Fatalf("failed to decode vector: %v", err) + } + vectorrunner.RunVector(t, "LoadConformance", "load", vector, vcSeam()) +} + +func TestVector79MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorJSON := `{ "name": "clear_by_category_and_all", "description": "Clear removes only the requested category, or all memories when no category is supplied.", @@ -1779,7 +2752,7 @@ func TestVector48MemoryConformanceOperateClearByCategoryAndAll(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector49MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { +func TestVector80MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T) { vectorJSON := `{ "name": "format_core_and_recall_results", "description": "Formatting injects only core memories into system prompts and formats recall results with category and tags.", @@ -1820,7 +2793,7 @@ func TestVector49MemoryConformanceOperateFormatCoreAndRecallResults(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector50MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { +func TestVector81MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) { vectorJSON := `{ "name": "recall_empty_query_and_limit", "description": "Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.", @@ -1874,7 +2847,7 @@ func TestVector50MemoryConformanceOperateRecallEmptyQueryAndLimit(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector51MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { +func TestVector82MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T) { vectorJSON := `{ "name": "recall_ranks_by_weighted_score", "description": "Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.", @@ -1932,7 +2905,7 @@ func TestVector51MemoryConformanceOperateRecallRanksByWeightedScore(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { +func TestVector83MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t *testing.T) { vectorJSON := `{ "name": "recall_score_ties_preserve_insertion_order", "description": "Recall preserves insertion order for memories with identical scores.", @@ -1989,7 +2962,7 @@ func TestVector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector53MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { +func TestVector84MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorJSON := `{ "name": "recall_tags_and_core_boost", "description": "Recall gives tag matches higher weight than content matches and boosts matching core memories.", @@ -2049,7 +3022,7 @@ func TestVector53MemoryConformanceOperateRecallTagsAndCoreBoost(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { +func TestVector85MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup(t *testing.T) { vectorJSON := `{ "name": "recall_tokenization_punctuation_unicode_and_dedup", "description": "Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.", @@ -2109,7 +3082,7 @@ func TestVector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAnd vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector55MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { +func TestVector86MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_by_tags", "description": "Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.", @@ -2172,7 +3145,7 @@ func TestVector55MemoryConformanceOperateRememberDedupsCoreByTags(t *testing.T) vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { +func TestVector87MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t *testing.T) { vectorJSON := `{ "name": "remember_dedups_core_missing_and_empty_tags", "description": "Remember treats missing and empty tag lists as equivalent for core-memory deduplication.", @@ -2227,7 +3200,7 @@ func TestVector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags(t vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector57MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { +func TestVector88MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing.T) { vectorJSON := `{ "name": "remember_evicts_archival_first", "description": "Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.", @@ -2287,7 +3260,7 @@ func TestVector57MemoryConformanceOperateRememberEvictsArchivalFirst(t *testing. vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector58MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { +func TestVector89MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorJSON := `{ "name": "snapshot_port_round_trip", "description": "A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.", @@ -2326,7 +3299,7 @@ func TestVector58MemoryConformanceOperateSnapshotPortRoundTrip(t *testing.T) { vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector59MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { +func TestVector90MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T) { vectorJSON := `{ "name": "update_index_out_of_range_error", "description": "Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.", @@ -2359,7 +3332,7 @@ func TestVector59MemoryConformanceOperateUpdateIndexOutOfRangeError(t *testing.T vectorrunner.RunVector(t, "MemoryConformance", "operate", vector, vcSeam()) } -func TestVector60TurnConformanceReplayMaxIterations(t *testing.T) { +func TestVector91TurnConformanceReplayMaxIterations(t *testing.T) { vectorJSON := `{ "name": "max_iterations", "stage": "replay", @@ -2395,7 +3368,7 @@ func TestVector60TurnConformanceReplayMaxIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector61TurnConformanceReplayNoTool(t *testing.T) { +func TestVector92TurnConformanceReplayNoTool(t *testing.T) { vectorJSON := `{ "name": "no_tool", "stage": "replay", @@ -2427,7 +3400,7 @@ func TestVector61TurnConformanceReplayNoTool(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector62TurnConformanceReplayPermissionDenied(t *testing.T) { +func TestVector93TurnConformanceReplayPermissionDenied(t *testing.T) { vectorJSON := `{ "name": "permission_denied", "stage": "replay", @@ -2462,7 +3435,7 @@ func TestVector62TurnConformanceReplayPermissionDenied(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector63TurnConformanceReplayToolFailure(t *testing.T) { +func TestVector94TurnConformanceReplayToolFailure(t *testing.T) { vectorJSON := `{ "name": "tool_failure", "stage": "replay", @@ -2499,7 +3472,7 @@ func TestVector63TurnConformanceReplayToolFailure(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector64TurnConformanceReplayToolSuccess(t *testing.T) { +func TestVector95TurnConformanceReplayToolSuccess(t *testing.T) { vectorJSON := `{ "name": "tool_success", "stage": "replay", @@ -2536,7 +3509,7 @@ func TestVector64TurnConformanceReplayToolSuccess(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "replay", vector, vcSeam()) } -func TestVector65TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { +func TestVector96TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorJSON := `{ "name": "assistant_tool_calls_metadata", "description": "When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.", @@ -2689,7 +3662,7 @@ func TestVector65TurnConformanceRunAssistantToolCallsMetadata(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector66TurnConformanceRunAsyncToolFunction(t *testing.T) { +func TestVector97TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorJSON := `{ "name": "async_tool_function", "description": "Agent loop correctly handles async tool functions", @@ -2800,7 +3773,7 @@ func TestVector66TurnConformanceRunAsyncToolFunction(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector67TurnConformanceRunBindingsInjected(t *testing.T) { +func TestVector98TurnConformanceRunBindingsInjected(t *testing.T) { vectorJSON := `{ "name": "bindings_injected", "description": "Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.", @@ -2931,7 +3904,7 @@ func TestVector67TurnConformanceRunBindingsInjected(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector68TurnConformanceRunCancellationBeforeLlm(t *testing.T) { +func TestVector99TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorJSON := `{ "name": "cancellation_before_llm", "description": "§13.2 Cancellation — Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.", @@ -2993,7 +3966,7 @@ func TestVector68TurnConformanceRunCancellationBeforeLlm(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector69TurnConformanceRunCancellationBetweenIterations(t *testing.T) { +func TestVector100TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_iterations", "description": "§13.2 Cancellation — Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.", @@ -3124,7 +4097,7 @@ func TestVector69TurnConformanceRunCancellationBetweenIterations(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector70TurnConformanceRunCancellationBetweenTools(t *testing.T) { +func TestVector101TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorJSON := `{ "name": "cancellation_between_tools", "description": "§13.2 Cancellation — LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.", @@ -3264,7 +4237,7 @@ func TestVector70TurnConformanceRunCancellationBetweenTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector71TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { +func TestVector102TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorJSON := `{ "name": "context_no_trim_when_fits", "description": "§13.3 Context Window — Messages fit within the context budget. No trimming occurs and all messages are preserved.", @@ -3336,7 +4309,7 @@ func TestVector71TurnConformanceRunContextNoTrimWhenFits(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector72TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { +func TestVector103TurnConformanceRunContextPreservesSystemMessages(t *testing.T) { vectorJSON := `{ "name": "context_preserves_system_messages", "description": "§13.3 Context Window — Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.", @@ -3444,7 +4417,7 @@ func TestVector72TurnConformanceRunContextPreservesSystemMessages(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector73TurnConformanceRunContextTrimBasic(t *testing.T) { +func TestVector104TurnConformanceRunContextTrimBasic(t *testing.T) { vectorJSON := `{ "name": "context_trim_basic", "description": "§13.3 Context Window — Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.", @@ -3603,7 +4576,7 @@ func TestVector73TurnConformanceRunContextTrimBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector74TurnConformanceRunEmptyToolResult(t *testing.T) { +func TestVector105TurnConformanceRunEmptyToolResult(t *testing.T) { vectorJSON := `{ "name": "empty_tool_result", "description": "Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content — it must not be skipped or filtered out.", @@ -3719,7 +4692,7 @@ func TestVector74TurnConformanceRunEmptyToolResult(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector75TurnConformanceRunEventsBasicToolLoop(t *testing.T) { +func TestVector106TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorJSON := `{ "name": "events_basic_tool_loop", "description": "§13.1 Events — A 2-turn tool-call loop emits the full event lifecycle: status → tool_call_start → tool_result → messages_updated → done.", @@ -3866,7 +4839,7 @@ func TestVector75TurnConformanceRunEventsBasicToolLoop(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector76TurnConformanceRunEventsErrorLogged(t *testing.T) { +func TestVector107TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorJSON := `{ "name": "events_error_logged", "description": "§13.1 Events — A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.", @@ -3993,7 +4966,7 @@ func TestVector76TurnConformanceRunEventsErrorLogged(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector77TurnConformanceRunEventsNoTools(t *testing.T) { +func TestVector108TurnConformanceRunEventsNoTools(t *testing.T) { vectorJSON := `{ "name": "events_no_tools", "description": "§13.1 Events — Single-turn completion with no tool calls. Only the done event is emitted.", @@ -4072,7 +5045,7 @@ func TestVector77TurnConformanceRunEventsNoTools(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector78TurnConformanceRunGuardrailAllPass(t *testing.T) { +func TestVector109TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorJSON := `{ "name": "guardrail_all_pass", "description": "§13.4 Guardrails — All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.", @@ -4196,7 +5169,7 @@ func TestVector78TurnConformanceRunGuardrailAllPass(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector79TurnConformanceRunGuardrailInputDeny(t *testing.T) { +func TestVector110TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_input_deny", "description": "§13.4 Guardrails — Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.", @@ -4253,7 +5226,7 @@ func TestVector79TurnConformanceRunGuardrailInputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector80TurnConformanceRunGuardrailOutputDeny(t *testing.T) { +func TestVector111TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_output_deny", "description": "§13.4 Guardrails — Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.", @@ -4332,7 +5305,7 @@ func TestVector80TurnConformanceRunGuardrailOutputDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector81TurnConformanceRunGuardrailToolDeny(t *testing.T) { +func TestVector112TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorJSON := `{ "name": "guardrail_tool_deny", "description": "§13.4 Guardrails — LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.", @@ -4487,7 +5460,7 @@ func TestVector81TurnConformanceRunGuardrailToolDeny(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector82TurnConformanceRunMaxIterationsExceeded(t *testing.T) { +func TestVector113TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorJSON := `{ "name": "max_iterations_exceeded", "description": "LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.", @@ -4998,7 +5971,7 @@ func TestVector82TurnConformanceRunMaxIterationsExceeded(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector83TurnConformanceRunMultiTurnToolCalls(t *testing.T) { +func TestVector114TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorJSON := `{ "name": "multi_turn_tool_calls", "description": "Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.", @@ -5176,7 +6149,7 @@ func TestVector83TurnConformanceRunMultiTurnToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector84TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { +func TestVector115TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorJSON := `{ "name": "multiple_tool_calls_single_turn", "description": "Turn 1: LLM returns two parallel tool calls — get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.", @@ -5358,7 +6331,7 @@ func TestVector84TurnConformanceRunMultipleToolCallsSingleTurn(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector85TurnConformanceRunNoToolCalls(t *testing.T) { +func TestVector116TurnConformanceRunNoToolCalls(t *testing.T) { vectorJSON := `{ "name": "no_tool_calls", "description": "LLM returns content immediately with no tool calls — the agent loop completes in a single iteration.", @@ -5427,7 +6400,7 @@ func TestVector85TurnConformanceRunNoToolCalls(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector86TurnConformanceRunParallelToolsBasic(t *testing.T) { +func TestVector117TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_basic", "description": "§13.6 Parallel Tools — LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.", @@ -5601,7 +6574,7 @@ func TestVector86TurnConformanceRunParallelToolsBasic(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector87TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { +func TestVector118TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_guardrail_deny", "description": "§13.6 Parallel Tools + §13.4 Guardrails — 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.", @@ -5793,7 +6766,7 @@ func TestVector87TurnConformanceRunParallelToolsWithGuardrailDeny(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector88TurnConformanceRunParallelToolsWithToolError(t *testing.T) { +func TestVector119TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorJSON := `{ "name": "parallel_tools_with_tool_error", "description": "§13.6 Parallel Tools + tool error — 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.", @@ -5967,7 +6940,7 @@ func TestVector88TurnConformanceRunParallelToolsWithToolError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector89TurnConformanceRunSingleToolCall(t *testing.T) { +func TestVector120TurnConformanceRunSingleToolCall(t *testing.T) { vectorJSON := `{ "name": "single_tool_call", "description": "Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.", @@ -6115,7 +7088,7 @@ func TestVector89TurnConformanceRunSingleToolCall(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector90TurnConformanceRunSteeringInjectMessage(t *testing.T) { +func TestVector121TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorJSON := `{ "name": "steering_inject_message", "description": "§13.5 Steering — After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.", @@ -6324,7 +7297,7 @@ func TestVector90TurnConformanceRunSteeringInjectMessage(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector91TurnConformanceRunSteeringMultipleMessages(t *testing.T) { +func TestVector122TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorJSON := `{ "name": "steering_multiple_messages", "description": "§13.5 Steering — Two steering messages are queued and both injected before iteration 2. Tests atomic drain — both appear in order.", @@ -6495,7 +7468,7 @@ func TestVector91TurnConformanceRunSteeringMultipleMessages(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector92TurnConformanceRunToolNotRegisteredError(t *testing.T) { +func TestVector123TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorJSON := `{ "name": "tool_not_registered_error", "description": "LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.", @@ -6580,7 +7553,7 @@ func TestVector92TurnConformanceRunToolNotRegisteredError(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector93TurnConformanceRunToolResultMessageFormat(t *testing.T) { +func TestVector124TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorJSON := `{ "name": "tool_result_message_format", "description": "After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.", @@ -6702,7 +7675,7 @@ func TestVector93TurnConformanceRunToolResultMessageFormat(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "run", vector, vcSeam()) } -func TestVector94TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { +func TestVector125TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorJSON := `{ "name": "cancel_before_context", "stage": "turn", @@ -6735,7 +7708,7 @@ func TestVector94TurnConformanceRunTurnCancelBeforeContext(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector95TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { +func TestVector126TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorJSON := `{ "name": "delegated_provider_state", "stage": "turn", @@ -6803,7 +7776,7 @@ func TestVector95TurnConformanceRunTurnDelegatedProviderState(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector96TurnConformanceRunTurnFinalOutput(t *testing.T) { +func TestVector127TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorJSON := `{ "name": "final_output", "stage": "turn", @@ -6849,7 +7822,7 @@ func TestVector96TurnConformanceRunTurnFinalOutput(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector97TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { +func TestVector128TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "memory_context_is_model_visible", "stage": "turn", @@ -6912,7 +7885,7 @@ func TestVector97TurnConformanceRunTurnMemoryContextIsModelVisible(t *testing.T) vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector98TurnConformanceRunTurnOrderedToolRound(t *testing.T) { +func TestVector129TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorJSON := `{ "name": "ordered_tool_round", "stage": "turn", @@ -7004,7 +7977,7 @@ func TestVector98TurnConformanceRunTurnOrderedToolRound(t *testing.T) { vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector99TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { +func TestVector130TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing.T) { vectorJSON := `{ "name": "permission_denial_is_model_visible", "stage": "turn", @@ -7056,7 +8029,7 @@ func TestVector99TurnConformanceRunTurnPermissionDenialIsModelVisible(t *testing vectorrunner.RunVector(t, "TurnConformance", "runTurn", vector, vcSeam()) } -func TestVector100WireConformanceToRequestAnthropicImageFormat(t *testing.T) { +func TestVector131WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorJSON := `{ "name": "anthropic_image_format", "description": "§7.5 — Anthropic: ImagePart uses base64 source block with media_type, not image_url format.", @@ -7127,7 +8100,7 @@ func TestVector100WireConformanceToRequestAnthropicImageFormat(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector101WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { +func TestVector132WireConformanceToRequestAnthropicMaxTokensRequired(t *testing.T) { vectorJSON := `{ "name": "anthropic_max_tokens_required", "description": "§7.5 — Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).", @@ -7183,7 +8156,7 @@ func TestVector101WireConformanceToRequestAnthropicMaxTokensRequired(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector102WireConformanceToRequestAnthropicOptions(t *testing.T) { +func TestVector133WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorJSON := `{ "name": "anthropic_options", "description": "Anthropic: option names map correctly (topK, stopSequences)", @@ -7253,7 +8226,7 @@ func TestVector102WireConformanceToRequestAnthropicOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector103WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { +func TestVector134WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) { vectorJSON := `{ "name": "anthropic_system_separate", "description": "§7.5 — Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.", @@ -7321,7 +8294,7 @@ func TestVector103WireConformanceToRequestAnthropicSystemSeparate(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector104WireConformanceToRequestAnthropicToolWire(t *testing.T) { +func TestVector135WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorJSON := `{ "name": "anthropic_tool_wire", "description": "Anthropic: tools use input_schema (not parameters), name at top level", @@ -7407,7 +8380,7 @@ func TestVector104WireConformanceToRequestAnthropicToolWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector105WireConformanceToRequestChatAudioMp3(t *testing.T) { +func TestVector136WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorJSON := `{ "name": "chat_audio_mp3", "description": "§7.1.2 — AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.", @@ -7466,7 +8439,7 @@ func TestVector105WireConformanceToRequestChatAudioMp3(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector106WireConformanceToRequestChatAudioPart(t *testing.T) { +func TestVector137WireConformanceToRequestChatAudioPart(t *testing.T) { vectorJSON := `{ "name": "chat_audio_part", "description": "§7.1.2 — AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.", @@ -7525,7 +8498,7 @@ func TestVector106WireConformanceToRequestChatAudioPart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector107WireConformanceToRequestChatImageBase64(t *testing.T) { +func TestVector138WireConformanceToRequestChatImageBase64(t *testing.T) { vectorJSON := `{ "name": "chat_image_base64", "description": "Base64 data URI image part preserved as-is in image_url", @@ -7590,7 +8563,7 @@ func TestVector107WireConformanceToRequestChatImageBase64(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector108WireConformanceToRequestChatImagePart(t *testing.T) { +func TestVector139WireConformanceToRequestChatImagePart(t *testing.T) { vectorJSON := `{ "name": "chat_image_part", "description": "Image content part converted to OpenAI image_url format", @@ -7655,7 +8628,7 @@ func TestVector108WireConformanceToRequestChatImagePart(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector109WireConformanceToRequestChatMultipartContent(t *testing.T) { +func TestVector140WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorJSON := `{ "name": "chat_multipart_content", "description": "§7.1.1/§7.1.2 — Message with TextPart + ImagePart MUST produce an array of typed content blocks.", @@ -7720,7 +8693,7 @@ func TestVector109WireConformanceToRequestChatMultipartContent(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector110WireConformanceToRequestChatSimple(t *testing.T) { +func TestVector141WireConformanceToRequestChatSimple(t *testing.T) { vectorJSON := `{ "name": "chat_simple", "description": "§7.1.7 — Two messages (system + user), no tools, no options. Minimal chat request.", @@ -7783,7 +8756,7 @@ func TestVector110WireConformanceToRequestChatSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector111WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { +func TestVector142WireConformanceToRequestChatSingleTextOptimized(t *testing.T) { vectorJSON := `{ "name": "chat_single_text_optimized", "description": "§7.1.1 — Message with exactly 1 TextPart MUST produce a plain string content, not an array.", @@ -7833,7 +8806,7 @@ func TestVector111WireConformanceToRequestChatSingleTextOptimized(t *testing.T) vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector112WireConformanceToRequestChatWithOptions(t *testing.T) { +func TestVector143WireConformanceToRequestChatWithOptions(t *testing.T) { vectorJSON := `{ "name": "chat_with_options", "description": "§7.1.5 — All standard ModelOptions mapped to OpenAI parameter names.", @@ -7902,7 +8875,7 @@ func TestVector112WireConformanceToRequestChatWithOptions(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector113WireConformanceToRequestEmbeddingWire(t *testing.T) { +func TestVector144WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorJSON := `{ "name": "embedding_wire", "description": "§7.2 — Embedding request: text extracted from messages, single string input (not array).", @@ -7947,7 +8920,7 @@ func TestVector113WireConformanceToRequestEmbeddingWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector114WireConformanceToRequestImageWire(t *testing.T) { +func TestVector145WireConformanceToRequestImageWire(t *testing.T) { vectorJSON := `{ "name": "image_wire", "description": "§7.3 — Image generation request: prompt extracted from last user message.", @@ -7992,7 +8965,7 @@ func TestVector114WireConformanceToRequestImageWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector115WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { +func TestVector146WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorJSON := `{ "name": "kind_to_json_type_mapping", "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.", @@ -8112,7 +9085,7 @@ func TestVector115WireConformanceToRequestKindToJsonTypeMapping(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector116WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { +func TestVector147WireConformanceToRequestOptionsAdditionalProperties(t *testing.T) { vectorJSON := `{ "name": "options_additional_properties", "description": "§7.1.5 — additionalProperties from ModelOptions MUST be merged into the request as top-level keys.", @@ -8167,7 +9140,7 @@ func TestVector116WireConformanceToRequestOptionsAdditionalProperties(t *testing vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector117WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { +func TestVector148WireConformanceToRequestOptionsMaxCompletionTokens(t *testing.T) { vectorJSON := `{ "name": "options_max_completion_tokens", "description": "§7.1.5 — maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.", @@ -8220,7 +9193,7 @@ func TestVector117WireConformanceToRequestOptionsMaxCompletionTokens(t *testing. vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector118WireConformanceToRequestOptionsStopSequences(t *testing.T) { +func TestVector149WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorJSON := `{ "name": "options_stop_sequences", "description": "§7.1.5 — stopSequences MUST map to 'stop' in the OpenAI wire format.", @@ -8279,7 +9252,7 @@ func TestVector118WireConformanceToRequestOptionsStopSequences(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector119WireConformanceToRequestResponsesSimple(t *testing.T) { +func TestVector150WireConformanceToRequestResponsesSimple(t *testing.T) { vectorJSON := `{ "name": "responses_simple", "description": "Responses API: system becomes instructions, messages become input items", @@ -8339,7 +9312,7 @@ func TestVector119WireConformanceToRequestResponsesSimple(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector120WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { +func TestVector151WireConformanceToRequestResponsesStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "responses_structured_output", "description": "Responses API: structured output uses text.format.json_schema", @@ -8427,7 +9400,7 @@ func TestVector120WireConformanceToRequestResponsesStructuredOutput(t *testing.T vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector121WireConformanceToRequestResponsesWithTools(t *testing.T) { +func TestVector152WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorJSON := `{ "name": "responses_with_tools", "description": "Responses API: tools use flat format (no nested function key)", @@ -8508,7 +9481,7 @@ func TestVector121WireConformanceToRequestResponsesWithTools(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector122WireConformanceToRequestStructuredOutput(t *testing.T) { +func TestVector153WireConformanceToRequestStructuredOutput(t *testing.T) { vectorJSON := `{ "name": "structured_output", "description": "§7.1.6 — outputs converted to response_format with json_schema, strict=true, additionalProperties=false.", @@ -8592,7 +9565,7 @@ func TestVector122WireConformanceToRequestStructuredOutput(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector123WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { +func TestVector154WireConformanceToRequestStructuredOutputNestedOptional(t *testing.T) { vectorJSON := `{ "name": "structured_output_nested_optional", "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", @@ -8695,7 +9668,7 @@ func TestVector123WireConformanceToRequestStructuredOutputNestedOptional(t *test vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector124WireConformanceToRequestToolsBindingsStripped(t *testing.T) { +func TestVector155WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorJSON := `{ "name": "tools_bindings_stripped", "description": "§7.1.3 — Parameters listed in bindings MUST be stripped from wire tools (properties AND required).", @@ -8788,7 +9761,7 @@ func TestVector124WireConformanceToRequestToolsBindingsStripped(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector125WireConformanceToRequestToolsFunctionWire(t *testing.T) { +func TestVector156WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorJSON := `{ "name": "tools_function_wire", "description": "§7.1.3 — FunctionTool projected as OpenAI function definition with JSON Schema parameters.", @@ -8871,7 +9844,7 @@ func TestVector125WireConformanceToRequestToolsFunctionWire(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector126WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { +func TestVector157WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorJSON := `{ "name": "tools_null_when_empty", "description": "§7.1.3 — When no tools are defined, the tools key MUST be absent from the request entirely.", @@ -8921,7 +9894,7 @@ func TestVector126WireConformanceToRequestToolsNullWhenEmpty(t *testing.T) { vectorrunner.RunVector(t, "WireConformance", "toRequest", vector, vcSeam()) } -func TestVector127WireConformanceToRequestToolsStrictMode(t *testing.T) { +func TestVector158WireConformanceToRequestToolsStrictMode(t *testing.T) { vectorJSON := `{ "name": "tools_strict_mode", "description": "§7.1.3 — FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.", diff --git a/runtime/go/prompty/model/vectorconformance/vector_conformance.go b/runtime/go/prompty/model/vectorconformance/vector_conformance.go index 81a5bbde0..c66decc53 100644 --- a/runtime/go/prompty/model/vectorconformance/vector_conformance.go +++ b/runtime/go/prompty/model/vectorconformance/vector_conformance.go @@ -1087,4 +1087,465 @@ func RunLoadConformanceConformance(t *testing.T, seam fixtures.LoadConformance) t.Fatalf("tools_openapi_load misrouted: got %#v, want %#v", actualValue, expectedValue) } } + // vector: trusted_endpoint_builtin_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_builtin_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_builtin_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_builtin_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_builtin_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_builtin_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_builtin_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_default_port_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_default_port_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_default_port_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_default_port_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_default_port_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_default_port_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_default_port_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_env_endpoint_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_env_endpoint_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_env_endpoint_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_env_endpoint_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_env_endpoint_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_env_endpoint_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_env_endpoint_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_env_list_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_env_list_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_env_list_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_env_list_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_env_list_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_env_list_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_env_list_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_literal_key_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_literal_key_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_literal_key_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_literal_key_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_literal_key_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_literal_key_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_literal_key_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_option_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_option_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_option_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_option_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_option_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_option_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_option_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_reference_skipped + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_reference_skipped: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_reference_skipped: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_reference_skipped: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_reference_skipped: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_reference_skipped: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_reference_skipped misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: trusted_endpoint_userinfo_allowed + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("trusted_endpoint_userinfo_allowed: decode source: %v", err) + } + actual, err := seam.Load(source) + if err != nil { + t.Fatalf("trusted_endpoint_userinfo_allowed: unexpected error: %v", err) + } + actualBytes, marshalErr := json.Marshal(actual) + if marshalErr != nil { + t.Fatalf("trusted_endpoint_userinfo_allowed: marshal actual: %v", marshalErr) + } + var actualValue any + if err := json.Unmarshal(actualBytes, &actualValue); err != nil { + t.Fatalf("trusted_endpoint_userinfo_allowed: decode actual: %v", err) + } + var expectedValue any + if err := json.Unmarshal([]byte("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}"), &expectedValue); err != nil { + t.Fatalf("trusted_endpoint_userinfo_allowed: decode expected: %v", err) + } + if !reflect.DeepEqual(actualValue, expectedValue) { + t.Fatalf("trusted_endpoint_userinfo_allowed misrouted: got %#v, want %#v", actualValue, expectedValue) + } + } + // vector: untrusted_endpoint_ambient_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_ambient_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_ambient_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_backslash_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_backslash_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_backslash_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_blank_literal_key_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_blank_literal_key_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_blank_literal_key_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_env_default_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_env_default_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_env_default_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_env_key_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_env_key_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_env_key_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_file_connection_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_file_connection_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_file_connection_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_fragment_before_at_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_fragment_before_at_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_fragment_before_at_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_ipv6_literal_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_ipv6_literal_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_ipv6_literal_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_key_with_client_secret_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_key_with_client_secret_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_key_with_client_secret_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_mismatched_credential_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_mismatched_credential_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_mismatched_credential_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_no_key_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_no_key_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_no_key_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_oauth_literal_secret_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_oauth_literal_secret_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_oauth_literal_secret_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_percent_encoded_host_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_percent_encoded_host_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_percent_encoded_host_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_port_mismatch_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_port_mismatch_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_port_mismatch_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_query_before_at_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_query_before_at_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_query_before_at_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_scheme_mismatch_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_scheme_mismatch_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_scheme_mismatch_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_suffix_lookalike_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_suffix_lookalike_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_suffix_lookalike_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_suffix_spoof_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_suffix_spoof_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_suffix_spoof_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_trailing_dot_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_trailing_dot_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_trailing_dot_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_unicode_homoglyph_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_unicode_homoglyph_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_unicode_homoglyph_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_userinfo_lookalike_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_userinfo_lookalike_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_userinfo_lookalike_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_wildcard_apex_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_wildcard_apex_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_wildcard_apex_rejected: expected an error, got nil") + } + } + // vector: untrusted_endpoint_wildcard_boundary_rejected + { + var source interface{} + if err := json.Unmarshal([]byte("null"), &source); err != nil { + t.Fatalf("untrusted_endpoint_wildcard_boundary_rejected: decode source: %v", err) + } + _, err := seam.Load(source) + if err == nil { + t.Fatalf("untrusted_endpoint_wildcard_boundary_rejected: expected an error, got nil") + } + } } diff --git a/runtime/go/prompty/vectoradapters/vectoradapters.go b/runtime/go/prompty/vectoradapters/vectoradapters.go index c23cad29d..310335bb6 100644 --- a/runtime/go/prompty/vectoradapters/vectoradapters.go +++ b/runtime/go/prompty/vectoradapters/vectoradapters.go @@ -412,6 +412,14 @@ func loadInvoke(input any, ctx Context) (any, error) { restore := applyEnv(typed["env"]) defer restore() + // Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + if envMap, _ := typed["env"].(map[string]any); envMap == nil || envMap[prompty.TrustedEndpointsEnv] == nil { + if prev, ok := os.LookupEnv(prompty.TrustedEndpointsEnv); ok { + os.Unsetenv(prompty.TrustedEndpointsEnv) + defer os.Setenv(prompty.TrustedEndpointsEnv, prev) + } + } + opts := prompty.LoadOptions{TrustedEndpoints: toStringSlice(typed["trustedEndpoints"])} // Input-validation vectors carry both a top-level `inputs` map and a // `frontmatter`; full-load vectors nest inputs inside the frontmatter. _, hasInputs := typed["inputs"] @@ -433,11 +441,11 @@ func loadInvoke(input any, ctx Context) (any, error) { switch { case isNonEmptyString(typed["fixture"]): dir := findSpecFixtures(ctx.BaseDir) - agent, err = prompty.LoadPromptyFile(filepath.Join(dir, typed["fixture"].(string))) + agent, err = prompty.LoadPromptyFileWithOptions(filepath.Join(dir, typed["fixture"].(string)), opts) case isNonEmptyString(typed["frontmatter_raw"]): - agent, err = loadFromRaw(typed["frontmatter_raw"].(string)) + agent, err = loadFromRaw(typed["frontmatter_raw"].(string), opts) case isMapAny(typed["frontmatter"]): - agent, err = materializeAndLoad(typed) + agent, err = materializeAndLoad(typed, opts) default: return nil, fmt.Errorf("") } @@ -473,19 +481,19 @@ func unwrapProperties(data map[string]any) map[string]any { return data } -func loadFromRaw(raw string) (prompty.Agent, error) { +func loadFromRaw(raw string, opts prompty.LoadOptions) (prompty.Agent, error) { tmp, err := os.MkdirTemp("", "prompty-load-") if err != nil { return prompty.Agent{}, err } defer os.RemoveAll(tmp) - return prompty.LoadPromptyContent(strings.ReplaceAll(raw, "\r\n", "\n"), tmp, []string{tmp}) + return prompty.LoadPromptyContentWithOptions(strings.ReplaceAll(raw, "\r\n", "\n"), tmp, []string{tmp}, opts) } // materializeAndLoad writes the vector's files (honoring agent_subdir and // allowing `..` keys so path-traversal vectors can plant a target outside the // allowed root), resolves references against the agent directory, and loads. -func materializeAndLoad(input map[string]any) (prompty.Agent, error) { +func materializeAndLoad(input map[string]any, opts prompty.LoadOptions) (prompty.Agent, error) { tempBase, err := os.MkdirTemp("", "prompty-load-") if err != nil { return prompty.Agent{}, err @@ -516,6 +524,9 @@ func materializeAndLoad(input map[string]any) (prompty.Agent, error) { frontmatter := toMapAny(input["frontmatter"]) roots := []string{canonicalDir(agentDir)} + if err := prompty.CheckEndpointTrust(frontmatter, agentDir, roots, opts.TrustedEndpoints); err != nil { + return prompty.Agent{}, err + } if err := prompty.ResolveReferences(frontmatter, agentDir, roots); err != nil { return prompty.Agent{}, err } @@ -647,6 +658,8 @@ func loadErrorPayload(err error) (map[string]any, bool) { kind = "invalid_template" case "required_input": kind = "missing_required_input" + case "untrusted_endpoint": + kind = "untrusted_endpoint" default: return nil, false } diff --git a/runtime/java/README.md b/runtime/java/README.md index 2d691db36..3ef0b6966 100644 --- a/runtime/java/README.md +++ b/runtime/java/README.md @@ -63,6 +63,20 @@ cd runtime/java ./gradlew :prompty:test # unit tests for the core module only ``` +## Endpoint trust + +A `.prompty` file is untrusted input. When its `model.connection` carries no +file-supplied credential (a literal `apiKey` on `kind: key`), executors fall back to ambient credentials, +so the loader refuses — with `LoadException.Kind.UNTRUSTED_ENDPOINT` — any +file-chosen `endpoint` / `tokenUrl` whose origin is not trusted (spec §2.5.1). +Built-in provider origins are always trusted; hosts add more through +`LoadOptions.trustedEndpoints` or the comma-separated +`PROMPTY_TRUSTED_ENDPOINTS` variable (`*` trusts everything): + +```java +Agent agent = Loader.load(path, LoadOptions.withTrustedEndpoints("http://localhost:11434")); +``` + ## Live provider tests Tests tagged `live` call real providers and are excluded from the default `test` diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java new file mode 100644 index 000000000..97d675b9d --- /dev/null +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/EndpointTrust.java @@ -0,0 +1,277 @@ +package com.microsoft.prompty; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +/** + * Endpoint trust — keeps ambient credentials on host-approved destinations (spec §2.5.1). + * + *

A {@code .prompty} file is untrusted input. When its connection carries no file-supplied + * credential, executors fall back to ambient credentials ({@code DefaultAzureCredential}, + * {@code *_API_KEY} environment variables). This class decides whether the file-chosen destination + * may receive them. + * + *

Trusted origins are the union of {@link #BUILTIN_TRUSTED_ENDPOINTS}, the host's + * {@link LoadOptions#trustedEndpoints()}, and the comma-separated {@value #TRUSTED_ENDPOINTS_ENV} + * variable (read through {@link Environment}). Frontmatter can never contribute to that set. + */ +public final class EndpointTrust { + + /** Environment variable listing additional trusted origins, comma-separated. */ + public static final String TRUSTED_ENDPOINTS_ENV = "PROMPTY_TRUSTED_ENDPOINTS"; + + /** First-party provider origins trusted without host configuration. */ + public static final List BUILTIN_TRUSTED_ENDPOINTS = + List.of( + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com"); + + private static final Map CREDENTIAL_KEYS = + Map.of("key", "apiKey"); + private static final List DESTINATION_KEYS = List.of("endpoint", "tokenUrl"); + + private EndpointTrust() {} + + /** An origin: lowercased scheme and host plus an explicit (or default) port. */ + record Origin(String scheme, String host, int port) {} + + /** + * Parse {@code url} into its origin, or {@code null} when it is not an absolute http(s) URL. + * + *

Hand-rolled rather than built on {@link java.net.URI}, which rejects hosts the other runtimes + * accept (underscores, for instance) and would make the same file trusted in one runtime and not + * another. The rules follow the reference runtime's {@code urlsplit}-based parsing. + */ + static Origin origin(String url) { + // HTTP clients treat '\' as '/', so parsers disagree on the host; reject outright. + if (url.indexOf('\\') >= 0) { + return null; + } + String s = url.strip().replace("\t", "").replace("\r", "").replace("\n", ""); + int colon = s.indexOf(':'); + if (colon <= 0 || !isScheme(s.substring(0, colon))) { + return null; + } + String scheme = s.substring(0, colon).toLowerCase(Locale.ROOT); + int defaultPort; + if ("http".equals(scheme)) { + defaultPort = 80; + } else if ("https".equals(scheme)) { + defaultPort = 443; + } else { + return null; + } + String rest = s.substring(colon + 1); + if (!rest.startsWith("//")) { + return null; + } + rest = rest.substring(2); + int end = rest.length(); + for (char delimiter : new char[] {'/', '?', '#'}) { + int index = rest.indexOf(delimiter); + if (index >= 0 && index < end) { + end = index; + } + } + String netloc = rest.substring(0, end); + boolean open = netloc.indexOf('[') >= 0; + boolean close = netloc.indexOf(']') >= 0; + if (open != close) { + return null; + } + + String hostinfo = netloc.substring(netloc.lastIndexOf('@') + 1); + String host; + String port; + int bracket = hostinfo.indexOf('['); + if (bracket >= 0) { + String bracketed = hostinfo.substring(bracket + 1); + int closing = bracketed.indexOf(']'); + host = closing < 0 ? bracketed : bracketed.substring(0, closing); + String after = closing < 0 ? "" : bracketed.substring(closing + 1); + int portColon = after.indexOf(':'); + port = portColon < 0 ? "" : after.substring(portColon + 1); + } else { + int portColon = hostinfo.indexOf(':'); + host = portColon < 0 ? hostinfo : hostinfo.substring(0, portColon); + port = portColon < 0 ? "" : hostinfo.substring(portColon + 1); + } + host = host.toLowerCase(Locale.ROOT); + if (host.isEmpty()) { + return null; + } + if (port.isEmpty()) { + return new Origin(scheme, host, defaultPort); + } + for (int i = 0; i < port.length(); i++) { + char c = port.charAt(i); + if (c < '0' || c > '9') { + return null; + } + } + int number; + try { + number = Integer.parseInt(port); + } catch (NumberFormatException e) { + return null; + } + if (number > 65535) { + return null; + } + return new Origin(scheme, host, number); + } + + private static boolean isScheme(String candidate) { + if (!Character.isLetter(candidate.charAt(0)) || candidate.charAt(0) > 127) { + return false; + } + for (int i = 0; i < candidate.length(); i++) { + char c = candidate.charAt(i); + boolean ok = + (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || c == '+' + || c == '-' + || c == '.'; + if (!ok) { + return false; + } + } + return true; + } + + private static List trustedEntries(List trustedEndpoints) { + List raw = new ArrayList<>(BUILTIN_TRUSTED_ENDPOINTS); + if (trustedEndpoints != null) { + raw.addAll(trustedEndpoints); + } + for (String part : Environment.lookup(TRUSTED_ENDPOINTS_ENV).orElse("").split(",", -1)) { + raw.add(part); + } + List entries = new ArrayList<>(); + for (String entry : raw) { + if (entry != null && !entry.strip().isEmpty()) { + entries.add(entry.strip()); + } + } + return entries; + } + + /** Whether {@code url}'s origin matches a built-in, host, or environment trusted origin. */ + public static boolean isTrustedEndpoint(String url) { + return isTrustedEndpoint(url, List.of()); + } + + /** + * Whether {@code url}'s origin matches a built-in origin, one of {@code trustedEndpoints}, or an + * entry of {@value #TRUSTED_ENDPOINTS_ENV}. + */ + public static boolean isTrustedEndpoint(String url, List trustedEndpoints) { + List entries = trustedEntries(trustedEndpoints); + if (entries.contains("*")) { + return true; + } + Origin target = url == null ? null : origin(url); + if (target == null) { + return false; + } + for (String entry : entries) { + Origin allowed = origin(entry); + if (allowed == null + || !allowed.scheme().equals(target.scheme()) + || allowed.port() != target.port()) { + continue; + } + String pattern = allowed.host(); + if (pattern.startsWith("*.")) { + if (target.host().endsWith(pattern.substring(1))) { + return true; + } + } else if (pattern.equals(target.host())) { + return true; + } + } + return false; + } + + static boolean isReference(String value) { + return value.startsWith("${") && value.endsWith("}"); + } + + private static boolean fileSuppliedCredential(Map connection) { + // Only the credential field the declared kind actually consumes counts; any other kind is + // ambient. + String key = connection.get("kind") instanceof String kind ? CREDENTIAL_KEYS.get(kind) : null; + if (key == null) { + return false; + } + // Blank keys count as absent: executors that trim fall back to ambient credentials. + return connection.get(key) instanceof String value && !value.isBlank() && !isReference(value); + } + + /** The effective destination if the file chose it, or {@code null} when the host did. */ + private static String fileControlledDestination(Object value) { + if (value == null) { + return null; + } + if (!(value instanceof String s)) { + return String.valueOf(value); + } + if (isReference(s)) { + String inner = s.substring(2, s.length() - 1); + int colon = inner.indexOf(':'); + String protocol = colon < 0 ? inner : inner.substring(0, colon); + if ("env".equals(protocol.toLowerCase(Locale.ROOT))) { + String rest = colon < 0 ? "" : inner.substring(colon + 1); + int defaultColon = rest.indexOf(':'); + String varName = defaultColon < 0 ? rest : rest.substring(0, defaultColon); + String defaultValue = defaultColon < 0 ? "" : rest.substring(defaultColon + 1); + Optional actual = Environment.lookup(varName); + if (actual.isPresent()) { + return null; + } + return defaultValue.isEmpty() ? null : defaultValue; + } + } + return s; + } + + /** + * Throw {@link LoadException.Kind#UNTRUSTED_ENDPOINT} if ambient credentials could reach a + * file-chosen, untrusted destination. + * + * @param connection the RAW (unresolved) {@code model.connection} value + * @param trustedEndpoints host-provided trusted origins; may be {@code null} + */ + public static void checkConnectionTrust(Object connection, List trustedEndpoints) { + if (!(connection instanceof Map map) || "reference".equals(map.get("kind"))) { + return; + } + if (fileSuppliedCredential(map)) { + return; + } + for (String key : DESTINATION_KEYS) { + String destination = fileControlledDestination(map.get(key)); + if (destination != null && !isTrustedEndpoint(destination, trustedEndpoints)) { + throw LoadException.untrustedEndpoint( + "Connection " + + key + + " '" + + destination + + "' is not trusted to receive ambient credentials. Set the " + + key + + " via ${env:VAR}, add its origin to trustedEndpoints or " + + TRUSTED_ENDPOINTS_ENV + + ", or use 'kind: reference'."); + } + } + } +} diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadException.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadException.java index e91556bfe..69da01f20 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadException.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadException.java @@ -27,6 +27,11 @@ public enum Kind { * {@code format} and {@code parser}. */ INVALID_TEMPLATE, + /** + * The connection would send ambient credentials to a destination the file chose and the host + * has not trusted (spec §2.5.1). + */ + UNTRUSTED_ENDPOINT, /** Any other load failure. */ OTHER } @@ -75,6 +80,10 @@ public static LoadException invalidTemplate(String detail) { return new LoadException(Kind.INVALID_TEMPLATE, "Invalid template format: " + detail); } + public static LoadException untrustedEndpoint(String detail) { + return new LoadException(Kind.UNTRUSTED_ENDPOINT, detail); + } + public static LoadException other(String detail) { return new LoadException(Kind.OTHER, "Load error: " + detail); } diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadOptions.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadOptions.java index 9e1a07064..fd345d787 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadOptions.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/LoadOptions.java @@ -8,13 +8,24 @@ * * @param allowedFileRoots additional directories that {@code ${file:...}} references may read from. * The prompt file's own directory is always allowed and need not be listed. + * @param trustedEndpoints additional origins (e.g. {@code "https://my-proxy.contoso.com"} or + * {@code "https://*.contoso.com"}) that may receive ambient credentials when a file chooses the + * connection endpoint (spec §2.5.1). Built-in provider origins and the + * {@code PROMPTY_TRUSTED_ENDPOINTS} variable are always consulted; {@code "*"} trusts every + * endpoint. */ -public record LoadOptions(List allowedFileRoots) { +public record LoadOptions(List allowedFileRoots, List trustedEndpoints) { - private static final LoadOptions DEFAULT = new LoadOptions(List.of()); + private static final LoadOptions DEFAULT = new LoadOptions(List.of(), List.of()); public LoadOptions { allowedFileRoots = List.copyOf(allowedFileRoots); + trustedEndpoints = trustedEndpoints == null ? List.of() : List.copyOf(trustedEndpoints); + } + + /** Options with additional {@code ${file:...}} roots and no extra trusted endpoints. */ + public LoadOptions(List allowedFileRoots) { + this(allowedFileRoots, List.of()); } /** Default options: {@code ${file:...}} is confined to the prompt's own directory tree. */ @@ -26,4 +37,14 @@ public static LoadOptions defaults() { public static LoadOptions withAllowedFileRoots(Path... roots) { return new LoadOptions(List.of(roots)); } + + /** Options trusting the given endpoint origins to receive ambient credentials. */ + public static LoadOptions withTrustedEndpoints(String... origins) { + return new LoadOptions(List.of(), List.of(origins)); + } + + /** A copy of these options with {@code trustedEndpoints} replaced. */ + public LoadOptions trustingEndpoints(List origins) { + return new LoadOptions(allowedFileRoots, origins); + } } diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/Loader.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/Loader.java index 15a015d6f..fcb991451 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/Loader.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/Loader.java @@ -23,6 +23,8 @@ *

  • Normalise authored shorthands — dictionary-form inputs and outputs whose value is a bare * scalar become full properties. *
  • Inject {@code kind: "prompt"}; a {@code .prompty} document always describes a prompt agent. + *
  • Check endpoint trust on the raw {@code model.connection} (spec §2.5.1): ambient credentials + * may only flow to a file-chosen destination whose origin the host trusts. *
  • Resolve {@code ${env:...}} and {@code ${file:...}} references. *
  • Hand the resulting map to the generated model's {@code load}. *
  • Record the source path in {@code metadata.__source_path}. @@ -111,6 +113,11 @@ private static com.microsoft.prompty.model.Agent buildAgent( data.put("kind", "prompt"); Path agentDir = filePath.getParent() == null ? Path.of(".") : filePath.getParent(); + + // Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. + EndpointTrust.checkConnectionTrust( + rawConnection(data, agentDir, options.allowedFileRoots()), options.trustedEndpoints()); + References.resolveReferences(data, agentDir, options.allowedFileRoots()); LoadContext context = makeLoadContext(agentDir, options.allowedFileRoots()); @@ -124,6 +131,31 @@ private static com.microsoft.prompty.model.Agent buildAgent( return agent; } + /** The unresolved {@code model.connection}, following whole-value {@code ${file:}} references. */ + private static Object rawConnection( + Map data, Path agentDir, List allowedFileRoots) { + Object model = followFileReference(data.get("model"), agentDir, allowedFileRoots, "model"); + if (!(model instanceof Map modelMap)) { + return null; + } + return followFileReference(modelMap.get("connection"), agentDir, allowedFileRoots, "connection"); + } + + private static Object followFileReference( + Object value, Path agentDir, List allowedFileRoots, String key) { + if (value instanceof String s && EndpointTrust.isReference(s)) { + String inner = s.substring(2, s.length() - 1); + int colon = inner.indexOf(':'); + String protocol = colon < 0 ? inner : inner.substring(0, colon); + if ("file".equals(protocol.toLowerCase(java.util.Locale.ROOT))) { + String reference = colon < 0 ? "" : inner.substring(colon + 1); + return References.readContainedFile(reference, agentDir, allowedFileRoots, key) + .orElse(null); + } + } + return value; + } + /** * A {@code LoadContext} whose {@code preProcess} resolves references as the model tree is walked. * diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/References.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/References.java index 9a8679fe0..1329ca8d6 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/References.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/References.java @@ -100,6 +100,22 @@ public static Optional resolveSingleRef( return tryResolveString(value, "", agentDir, allowedFileRoots, false); } + /** + * Read a {@code ${file:...}} target under the same containment rules as reference resolution. + * + *

    A target that escapes the allowed roots throws {@link LoadException.Kind#FILE_REFERENCE}; a + * contained target that does not exist yields empty, leaving the error to normal resolution. + */ + static Optional readContainedFile( + String relativePath, Path agentDir, List allowedFileRoots, String key) { + Path requested = Path.of(relativePath); + Path full = requested.isAbsolute() ? requested : agentDir.resolve(requested); + if (!Files.exists(full)) { + return Optional.empty(); + } + return resolveFile(relativePath, agentDir, allowedFileRoots, key); + } + private static Optional tryResolveString( String value, String key, Path agentDir, List allowedFileRoots, boolean throwOnError) { if (!value.startsWith("${") || !value.endsWith("}")) { diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java index ffef595c3..261bf926d 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/model/VectorConformance.java @@ -246,7 +246,7 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex { // vector: connection_types_load StringBuilder inputJson = new StringBuilder(); - inputJson.append("{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}}"); + inputJson.append("{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"trustedEndpoints\":[\"https://localhost:8080\"]}"); java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); Object source = (Object) input.get("source"); Agent actual = seam.load(source); @@ -285,7 +285,7 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex { // vector: env_default StringBuilder inputJson = new StringBuilder(); - inputJson.append("{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}}"); + inputJson.append("{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\",\"apiKey\":\"literal-key\"}}},\"env\":{}}"); java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); Object source = (Object) input.get("source"); Agent actual = seam.load(source); @@ -600,6 +600,432 @@ public static void runLoadConformanceConformance(LoadConformance seam) throws Ex if (!actual.toJson().equals(TypraJson.stringify(expected))) throw new AssertionError("tools_openapi_load misrouted"); } + { + // vector: trusted_endpoint_builtin_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-builtin\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_builtin_allowed misrouted"); + } + { + // vector: trusted_endpoint_default_port_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-default-port-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_default_port_allowed misrouted"); + } + { + // vector: trusted_endpoint_env_endpoint_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_env_endpoint_allowed misrouted"); + } + { + // vector: trusted_endpoint_env_list_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_env_list_allowed misrouted"); + } + { + // vector: trusted_endpoint_literal_key_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_literal_key_allowed misrouted"); + } + { + // vector: trusted_endpoint_option_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_option_allowed misrouted"); + } + { + // vector: trusted_endpoint_reference_skipped + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_reference_skipped misrouted"); + } + { + // vector: trusted_endpoint_userinfo_allowed + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"trusted-userinfo-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Agent actual = seam.load(source); + StringBuilder expectedJson = new StringBuilder(); + expectedJson.append("{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}"); + Object expected = TypraJson.parse(expectedJson.toString()); + if (!actual.toJson().equals(TypraJson.stringify(expected))) + throw new AssertionError("trusted_endpoint_userinfo_allowed misrouted"); + } + { + // vector: untrusted_endpoint_ambient_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_ambient_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_backslash_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_backslash_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_blank_literal_key_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-blank-key\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\" \"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_blank_literal_key_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_env_default_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_env_default_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_env_key_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_env_key_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_file_connection_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_file_connection_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_fragment_before_at_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_fragment_before_at_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_ipv6_literal_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_ipv6_literal_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_key_with_client_secret_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_key_with_client_secret_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_mismatched_credential_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_mismatched_credential_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_no_key_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_no_key_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_oauth_literal_secret_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_oauth_literal_secret_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_percent_encoded_host_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_percent_encoded_host_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_port_mismatch_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_port_mismatch_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_query_before_at_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_query_before_at_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_scheme_mismatch_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_scheme_mismatch_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_suffix_lookalike_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_suffix_lookalike_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_suffix_spoof_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_suffix_spoof_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_trailing_dot_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_trailing_dot_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_unicode_homoglyph_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\\u0430pi.openai.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_unicode_homoglyph_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_userinfo_lookalike_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_userinfo_lookalike_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_wildcard_apex_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_wildcard_apex_rejected: expected an error"); + } + { + // vector: untrusted_endpoint_wildcard_boundary_rejected + StringBuilder inputJson = new StringBuilder(); + inputJson.append("{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}}"); + java.util.Map input = (java.util.Map) TypraJson.parse(inputJson.toString()); + Object source = (Object) input.get("source"); + Throwable caught = null; + try { + seam.load(source); + } catch (Throwable error) { + caught = error; + } + if (caught == null) throw new AssertionError("untrusted_endpoint_wildcard_boundary_rejected: expected an error"); + } } } diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/EndpointTrustTest.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/EndpointTrustTest.java new file mode 100644 index 000000000..5eb7dd29e --- /dev/null +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/EndpointTrustTest.java @@ -0,0 +1,257 @@ +package com.microsoft.prompty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.microsoft.prompty.model.Agent; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** Endpoint trust (spec §2.5.1). */ +class EndpointTrustTest { + + private static final String ENV = EndpointTrust.TRUSTED_ENDPOINTS_ENV; + + @TempDir Path tmp; + + @BeforeEach + void maskEnv() { + Environment.mask(ENV); + } + + @AfterEach + void clearEnv() { + Environment.clear(ENV); + } + + private Path write(String connection) throws IOException { + Path p = tmp.resolve("t.prompty"); + Files.writeString( + p, + "---\nname: t\nmodel:\n id: gpt-4\n connection:\n" + connection + "\n---\nHello\n", + StandardCharsets.UTF_8); + return p; + } + + private static LoadException assertUntrusted(Path p, LoadOptions options) { + LoadException e = assertThrows(LoadException.class, () -> Loader.load(p, options)); + assertEquals(LoadException.Kind.UNTRUSTED_ENDPOINT, e.kind()); + return e; + } + + private static LoadException assertUntrusted(Path p) { + return assertUntrusted(p, LoadOptions.defaults()); + } + + private static java.util.Map save(Agent agent) { + return agent.model.connection.save(new com.microsoft.prompty.model.SaveContext()); + } + + @Nested + class IsTrustedEndpoint { + + @ParameterizedTest + @ValueSource( + strings = { + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + }) + void builtinTrusted(String url) { + assertTrue(EndpointTrust.isTrustedEndpoint(url)); + } + + @ParameterizedTest + @ValueSource( + strings = { + "http://api.openai.com", // scheme mismatch + "https://api.openai.com:8443", // port mismatch + "https://openai.azure.com", // wildcard never matches apex + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", // userinfo ignored; host is evil.com + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "", + }) + void untrusted(String url) { + assertFalse(EndpointTrust.isTrustedEndpoint(url)); + } + + @Test + void hostOptionAndWildcard() { + assertTrue( + EndpointTrust.isTrustedEndpoint("http://localhost:11434/v1", List.of("http://LOCALHOST:11434"))); + assertTrue(EndpointTrust.isTrustedEndpoint("https://x.contoso.com", List.of("https://*.contoso.com"))); + assertFalse(EndpointTrust.isTrustedEndpoint("https://contoso.com", List.of("https://*.contoso.com"))); + } + + @Test + void hostWithUnderscoreParses() { + assertTrue( + EndpointTrust.isTrustedEndpoint("https://my_host.example.com/x", List.of("https://my_host.example.com"))); + } + + @Test + void envListAndStar() { + Environment.set(ENV, " https://a.example.com , http://localhost:8080"); + assertTrue(EndpointTrust.isTrustedEndpoint("http://localhost:8080/x")); + assertFalse(EndpointTrust.isTrustedEndpoint("http://localhost:8081")); + Environment.set(ENV, "*"); + assertTrue(EndpointTrust.isTrustedEndpoint("http://anything.example")); + } + } + + @Nested + class LoaderEnforcement { + + @Test + void ambientLiteralEndpointRejected() throws IOException { + LoadException e = assertUntrusted(write(" kind: foundry\n endpoint: http://127.0.0.1:9999")); + assertTrue(e.getMessage().contains("127.0.0.1")); + } + + @Test + void envKeyLiteralEndpointRejected() throws IOException { + Environment.set("SOME_KEY", "secret"); + try { + LoadException e = + assertUntrusted( + write(" kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}")); + assertTrue(e.getMessage().contains("not trusted")); + } finally { + Environment.clear("SOME_KEY"); + } + } + + @Test + void oauthTokenUrlChecked() throws IOException { + Environment.set("SECRET", "s"); + try { + LoadException e = + assertUntrusted( + write( + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token")); + assertTrue(e.getMessage().contains("tokenUrl")); + } finally { + Environment.clear("SECRET"); + } + } + + @Test + void mismatchedCredentialFieldRejected() throws IOException { + assertUntrusted(write(" kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk")); + assertUntrusted(write(" kind: key\n endpoint: https://evil.example.com\n clientSecret: junk")); + } + + @Test + void literalKeyAllowed() throws IOException { + Agent agent = Loader.load(write(" kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine")); + assertEquals( + "http://127.0.0.1:9999", + ((com.microsoft.prompty.model.ApiKeyConnection) agent.model.connection).endpoint); + } + + @Test + void envEndpointAllowed() throws IOException { + Environment.set("EP", "http://127.0.0.1:9999"); + try { + Agent agent = Loader.load(write(" kind: foundry\n endpoint: ${env:EP}")); + assertEquals("http://127.0.0.1:9999", save(agent).get("endpoint")); + } finally { + Environment.clear("EP"); + } + } + + @Test + void envDefaultUsedRejected() throws IOException { + Environment.mask("UNSET_EP"); + try { + assertUntrusted(write(" kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}")); + } finally { + Environment.clear("UNSET_EP"); + } + } + + @Test + void trustedOptionAllows() throws IOException { + Path p = write(" kind: foundry\n endpoint: http://127.0.0.1:9999/api"); + Agent agent = Loader.load(p, LoadOptions.withTrustedEndpoints("http://127.0.0.1:9999")); + assertEquals("http://127.0.0.1:9999/api", save(agent).get("endpoint")); + } + + @Test + void loadFromStringHonorsOption() { + String raw = + "---\nname: t\nmodel:\n id: gpt-4\n connection:\n kind: foundry\n" + + " endpoint: http://127.0.0.1:9999\n---\nHi\n"; + Path base = tmp.resolve("virtual.prompty"); + LoadException e = assertThrows(LoadException.class, () -> Loader.loadFromString(raw, base)); + assertEquals(LoadException.Kind.UNTRUSTED_ENDPOINT, e.kind()); + Agent agent = + Loader.loadFromString(raw, base, LoadOptions.withTrustedEndpoints("http://127.0.0.1:9999")); + assertEquals("http://127.0.0.1:9999", save(agent).get("endpoint")); + } + + @Test + void fileModelReferenceChecked() throws IOException { + Files.writeString( + tmp.resolve("model.json"), + "{\"id\": \"gpt-4\", \"connection\": {\"kind\": \"foundry\", \"endpoint\": \"https://evil.example.com\"}}", + StandardCharsets.UTF_8); + Path p = tmp.resolve("t.prompty"); + Files.writeString(p, "---\nname: t\nmodel: ${file:model.json}\n---\nHi\n", StandardCharsets.UTF_8); + LoadException e = assertUntrusted(p); + assertTrue(e.getMessage().contains("not trusted")); + } + + @Test + void traversalFileConnectionRaisesFileReference() throws IOException { + Path sub = Files.createDirectories(tmp.resolve("agent")); + Files.writeString( + tmp.resolve("conn.json"), + "{\"kind\": \"foundry\", \"endpoint\": \"https://evil.example.com\"}", + StandardCharsets.UTF_8); + Path p = sub.resolve("t.prompty"); + Files.writeString( + p, + "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:../conn.json}\n---\nHi\n", + StandardCharsets.UTF_8); + LoadException e = assertThrows(LoadException.class, () -> Loader.load(p)); + assertEquals(LoadException.Kind.FILE_REFERENCE, e.kind()); + } + + @Test + void frontmatterCannotSelfTrust() throws IOException { + Path p = tmp.resolve("t.prompty"); + Files.writeString( + p, + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n", + StandardCharsets.UTF_8); + assertUntrusted(p); + } + + @Test + void referenceKindSkipped() throws IOException { + Agent agent = Loader.load(write(" kind: reference\n name: my-connection")); + assertEquals("reference", save(agent).get("kind")); + } + } +} diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/LoadVectorsTest.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/LoadVectorsTest.java index b16d93937..63df92124 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/LoadVectorsTest.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/LoadVectorsTest.java @@ -142,6 +142,7 @@ private static Map canonicalError(Throwable thrown, Agent agent, case ENV_VAR_NOT_SET -> "env_var_not_set"; case FILE_REFERENCE -> "file_reference"; case INVALID_TEMPLATE -> "invalid_template"; + case UNTRUSTED_ENDPOINT -> "untrusted_endpoint"; case OTHER -> "other"; }); } else if (thrown instanceof InvokerException invoker @@ -184,7 +185,7 @@ private static String firstMissingRequired(Agent agent, Map prov private Agent load(Map input) { String fixture = SpecVectors.string(input, "fixture"); if (fixture != null) { - return Loader.load(SpecVectors.fixtures().resolve(fixture)); + return Loader.load(SpecVectors.fixtures().resolve(fixture), loadOptions(input)); } String raw = SpecVectors.string(input, "frontmatter_raw"); @@ -217,7 +218,7 @@ private Agent load(Map input) { // `${file:}` references resolve relative to the agent's own directory, so the vector's virtual // files have to sit beside a virtual agent path rather than beside the test's working directory. - return Loader.loadFromString(raw, agentDir.resolve("virtual.prompty")); + return Loader.loadFromString(raw, agentDir.resolve("virtual.prompty"), loadOptions(input)); } private static String toYaml(Object frontmatter) { @@ -273,9 +274,25 @@ private static List setEnv(Map input, Map input) { + List trusted = new ArrayList<>(); + if (input.get("trustedEndpoints") instanceof List list) { + for (Object item : list) { + trusted.add(String.valueOf(item)); + } + } + return new LoadOptions(List.of(), trusted); + } + private static void clearEnv(List keys) { for (String key : keys) { Environment.clear(key); diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java index 7b8d29bfb..1f8facb13 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorAdapters.java @@ -2,9 +2,11 @@ import com.microsoft.prompty.CancellationToken; import com.microsoft.prompty.Discovery; +import com.microsoft.prompty.EndpointTrust; import com.microsoft.prompty.Environment; import com.microsoft.prompty.InvokerException; import com.microsoft.prompty.LoadException; +import com.microsoft.prompty.LoadOptions; import com.microsoft.prompty.Loader; import com.microsoft.prompty.Memory; import com.microsoft.prompty.Messages; @@ -319,6 +321,7 @@ private static Map loadKindPayload(LoadException e) { case ENV_VAR_NOT_SET -> "env_var_not_set"; case FILE_REFERENCE -> "file_reference"; case INVALID_TEMPLATE -> "invalid_template"; + case UNTRUSTED_ENDPOINT -> "untrusted_endpoint"; case OTHER -> "other"; }); return payload; @@ -349,7 +352,7 @@ private static String firstMissingRequired(Agent agent, Map prov private static Agent loadAgent(Map input) { String fixture = string(input.get("fixture")); if (!fixture.isEmpty()) { - return Loader.load(SpecVectors.fixtures().resolve(fixture)); + return Loader.load(SpecVectors.fixtures().resolve(fixture), loadOptions(input)); } String raw = input.get("frontmatter_raw") instanceof String r ? r : null; Map files = asMap(input.get("files")); @@ -371,7 +374,7 @@ private static Agent loadAgent(Map input) { Object content = file.getValue(); write(target, content instanceof String text ? text : TypraJson.stringify(content)); } - return Loader.loadFromString(raw, agentDir.resolve("virtual.prompty")); + return Loader.loadFromString(raw, agentDir.resolve("virtual.prompty"), loadOptions(input)); } private static String toYaml(Object frontmatter) { @@ -420,9 +423,25 @@ private static List setEnv(Map input, Map input) { + List trusted = new ArrayList<>(); + if (input.get("trustedEndpoints") instanceof List list) { + for (Object item : list) { + trusted.add(String.valueOf(item)); + } + } + return new LoadOptions(List.of(), trusted); + } + private static void clearEnv(List keys) { for (String key : keys) { Environment.clear(key); diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java index cd3518f3e..72d2084a6 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/model/VectorConformanceTests.java @@ -206,7 +206,7 @@ private static void vector20LoadConformanceLoadBasicLoad() throws Exception { private static void vector21LoadConformanceLoadConnectionTypesLoad() throws Exception { StringBuilder sb = new StringBuilder(); - sb.append("{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}"); + sb.append("{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"trustedEndpoints\":[\"https://localhost:8080\"]},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } @@ -227,7 +227,7 @@ private static void vector23LoadConformanceLoadEmptyFrontmatterBodyOnly() throws private static void vector24LoadConformanceLoadEnvDefault() throws Exception { StringBuilder sb = new StringBuilder(); - sb.append("{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}"); + sb.append("{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\",\"apiKey\":\"literal-key\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } @@ -393,560 +393,777 @@ private static void vector47LoadConformanceLoadToolsOpenapiLoad() throws Excepti VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); } - private static void vector48MemoryConformanceOperateClearByCategoryAndAll() throws Exception { + private static void vector48LoadConformanceLoadTrustedEndpointBuiltinAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_builtin_allowed\",\"description\":\"Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-builtin\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector49LoadConformanceLoadTrustedEndpointDefaultPortAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_default_port_allowed\",\"description\":\"An explicit default port normalizes to the built-in origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-default-port-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector50LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_env_endpoint_allowed\",\"description\":\"An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector51LoadConformanceLoadTrustedEndpointEnvListAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_env_list_allowed\",\"description\":\"PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector52LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_literal_key_allowed\",\"description\":\"A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector53LoadConformanceLoadTrustedEndpointOptionAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_option_allowed\",\"description\":\"A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector54LoadConformanceLoadTrustedEndpointReferenceSkipped() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_reference_skipped\",\"description\":\"kind: reference connections carry no endpoint and are not subject to the trust check\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector55LoadConformanceLoadTrustedEndpointUserinfoAllowed() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"trusted_endpoint_userinfo_allowed\",\"description\":\"Userinfo is ignored; the destination host api.openai.com is trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-userinfo-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector56LoadConformanceLoadUntrustedEndpointAmbientRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_ambient_rejected\",\"description\":\"A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec \\u00a72.5.1)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector57LoadConformanceLoadUntrustedEndpointBackslashRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_backslash_rejected\",\"description\":\"A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector58LoadConformanceLoadUntrustedEndpointBlankLiteralKeyRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_blank_literal_key_rejected\",\"description\":\"a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-blank-key\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\" \"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector59LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector60LoadConformanceLoadUntrustedEndpointEnvKeyRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector61LoadConformanceLoadUntrustedEndpointFileConnectionRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector62LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_fragment_before_at_rejected\",\"description\":\"A '#' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector63LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_ipv6_literal_rejected\",\"description\":\"An IPv6 literal is not a trusted origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector64LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector65LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector66LoadConformanceLoadUntrustedEndpointNoKeyRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector67LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_oauth_literal_secret_rejected\",\"description\":\"kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector68LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_percent_encoded_host_rejected\",\"description\":\"Hosts are compared without percent-decoding, so an encoded host never matches\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector69LoadConformanceLoadUntrustedEndpointPortMismatchRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_port_mismatch_rejected\",\"description\":\"A non-default port is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector70LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_query_before_at_rejected\",\"description\":\"A '?' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector71LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_scheme_mismatch_rejected\",\"description\":\"Built-in origins are https; http on the same host is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector72LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_suffix_lookalike_rejected\",\"description\":\"A trusted host as a label prefix of another domain is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector73LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector74LoadConformanceLoadUntrustedEndpointTrailingDotRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_trailing_dot_rejected\",\"description\":\"Hosts are compared exactly; a trailing-dot host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector75LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_unicode_homoglyph_rejected\",\"description\":\"A non-ASCII lookalike host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\\u0430pi.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector76LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_userinfo_lookalike_rejected\",\"description\":\"A trusted host in userinfo is not the destination host\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector77LoadConformanceLoadUntrustedEndpointWildcardApexRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector78LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"untrusted_endpoint_wildcard_boundary_rejected\",\"description\":\"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}"); + Object vector = TypraJson.parse(sb.toString()); + VectorRunner.runVector("LoadConformance", "load", vector, false, seam()); + } + + private static void vector79MemoryConformanceOperateClearByCategoryAndAll() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector49MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { + private static void vector80MemoryConformanceOperateFormatCoreAndRecallResults() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector50MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { + private static void vector81MemoryConformanceOperateRecallEmptyQueryAndLimit() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector51MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { + private static void vector82MemoryConformanceOperateRecallRanksByWeightedScore() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { + private static void vector83MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector53MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { + private static void vector84MemoryConformanceOperateRecallTagsAndCoreBoost() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { + private static void vector85MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\\u00e9\"]},{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\\u00e9\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\\u00e9\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector55MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { + private static void vector86MemoryConformanceOperateRememberDedupsCoreByTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { + private static void vector87MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector57MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { + private static void vector88MemoryConformanceOperateRememberEvictsArchivalFirst() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector58MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { + private static void vector89MemoryConformanceOperateSnapshotPortRoundTrip() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector59MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { + private static void vector90MemoryConformanceOperateUpdateIndexOutOfRangeError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("MemoryConformance", "operate", vector, false, seam()); } - private static void vector60TurnConformanceReplayMaxIterations() throws Exception { + private static void vector91TurnConformanceReplayMaxIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector61TurnConformanceReplayNoTool() throws Exception { + private static void vector92TurnConformanceReplayNoTool() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector62TurnConformanceReplayPermissionDenied() throws Exception { + private static void vector93TurnConformanceReplayPermissionDenied() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector63TurnConformanceReplayToolFailure() throws Exception { + private static void vector94TurnConformanceReplayToolFailure() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector64TurnConformanceReplayToolSuccess() throws Exception { + private static void vector95TurnConformanceReplayToolSuccess() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "replay", vector, false, seam()); } - private static void vector65TurnConformanceRunAssistantToolCallsMetadata() throws Exception { + private static void vector96TurnConformanceRunAssistantToolCallsMetadata() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector66TurnConformanceRunAsyncToolFunction() throws Exception { + private static void vector97TurnConformanceRunAsyncToolFunction() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector67TurnConformanceRunBindingsInjected() throws Exception { + private static void vector98TurnConformanceRunBindingsInjected() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\\u00b0C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\\u00b0C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector68TurnConformanceRunCancellationBeforeLlm() throws Exception { + private static void vector99TurnConformanceRunCancellationBeforeLlm() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_before_llm\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector69TurnConformanceRunCancellationBetweenIterations() throws Exception { + private static void vector100TurnConformanceRunCancellationBetweenIterations() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_iterations\",\"description\":\"\\u00a713.2 Cancellation \\u2014 Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector70TurnConformanceRunCancellationBetweenTools() throws Exception { + private static void vector101TurnConformanceRunCancellationBetweenTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancellation_between_tools\",\"description\":\"\\u00a713.2 Cancellation \\u2014 LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\\u00b0F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector71TurnConformanceRunContextNoTrimWhenFits() throws Exception { + private static void vector102TurnConformanceRunContextNoTrimWhenFits() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector72TurnConformanceRunContextPreservesSystemMessages() throws Exception { + private static void vector103TurnConformanceRunContextPreservesSystemMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_preserves_system_messages\",\"description\":\"\\u00a713.3 Context Window \\u2014 Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \\u2014 they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector73TurnConformanceRunContextTrimBasic() throws Exception { + private static void vector104TurnConformanceRunContextTrimBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"context_trim_basic\",\"description\":\"\\u00a713.3 Context Window \\u2014 Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector74TurnConformanceRunEmptyToolResult() throws Exception { + private static void vector105TurnConformanceRunEmptyToolResult() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \\u2014 it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector75TurnConformanceRunEventsBasicToolLoop() throws Exception { + private static void vector106TurnConformanceRunEventsBasicToolLoop() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_basic_tool_loop\",\"description\":\"\\u00a713.1 Events \\u2014 A 2-turn tool-call loop emits the full event lifecycle: status \\u2192 tool_call_start \\u2192 tool_result \\u2192 messages_updated \\u2192 done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\\u00b0F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector76TurnConformanceRunEventsErrorLogged() throws Exception { + private static void vector107TurnConformanceRunEventsErrorLogged() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_error_logged\",\"description\":\"\\u00a713.1 Events \\u0097 A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \\u0097 this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector77TurnConformanceRunEventsNoTools() throws Exception { + private static void vector108TurnConformanceRunEventsNoTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"events_no_tools\",\"description\":\"\\u00a713.1 Events \\u2014 Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector78TurnConformanceRunGuardrailAllPass() throws Exception { + private static void vector109TurnConformanceRunGuardrailAllPass() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_all_pass\",\"description\":\"\\u00a713.4 Guardrails \\u2014 All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector79TurnConformanceRunGuardrailInputDeny() throws Exception { + private static void vector110TurnConformanceRunGuardrailInputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_input_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector80TurnConformanceRunGuardrailOutputDeny() throws Exception { + private static void vector111TurnConformanceRunGuardrailOutputDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_output_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector81TurnConformanceRunGuardrailToolDeny() throws Exception { + private static void vector112TurnConformanceRunGuardrailToolDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"guardrail_tool_deny\",\"description\":\"\\u00a713.4 Guardrails \\u2014 LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \\u2014 its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector82TurnConformanceRunMaxIterationsExceeded() throws Exception { + private static void vector113TurnConformanceRunMaxIterationsExceeded() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\\u00b0F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\\u00b0F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\\u00b0F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\\u00b0F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\\u00b0F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\\u00b0F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\\u00b0F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\\u00b0F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\\u00b0F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector83TurnConformanceRunMultiTurnToolCalls() throws Exception { + private static void vector114TurnConformanceRunMultiTurnToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\\u00b0C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\\u00b0C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector84TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { + private static void vector115TurnConformanceRunMultipleToolCallsSingleTurn() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \\u2014 get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\\u00b0F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\\u00b0F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny, while London is 59\\u00b0F and cloudy.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector85TurnConformanceRunNoToolCalls() throws Exception { + private static void vector116TurnConformanceRunNoToolCalls() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \\u2014 the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector86TurnConformanceRunParallelToolsBasic() throws Exception { + private static void vector117TurnConformanceRunParallelToolsBasic() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_basic\",\"description\":\"\\u00a713.6 Parallel Tools \\u2014 LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector87TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { + private static void vector118TurnConformanceRunParallelToolsWithGuardrailDeny() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\\u00a713.6 Parallel Tools + \\u00a713.4 Guardrails \\u2014 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector88TurnConformanceRunParallelToolsWithToolError() throws Exception { + private static void vector119TurnConformanceRunParallelToolsWithToolError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\\u00a713.6 Parallel Tools + tool error \\u2014 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\\u00b0F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\\u00b0F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector89TurnConformanceRunSingleToolCall() throws Exception { + private static void vector120TurnConformanceRunSingleToolCall() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\\u00b0F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\\u00b0F and sunny.\"}]},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector90TurnConformanceRunSteeringInjectMessage() throws Exception { + private static void vector121TurnConformanceRunSteeringInjectMessage() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_inject_message\",\"description\":\"\\u00a713.5 Steering \\u2014 After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\\u00b0F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\\u00b0C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector91TurnConformanceRunSteeringMultipleMessages() throws Exception { + private static void vector122TurnConformanceRunSteeringMultipleMessages() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"steering_multiple_messages\",\"description\":\"\\u00a713.5 Steering \\u2014 Two steering messages are queued and both injected before iteration 2. Tests atomic drain \\u2014 both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\\u00b0F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\\u00b0F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \\u00e0 Paris est de 22\\u00b0C et ensoleill\\u00e9 avec une humidit\\u00e9 de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector92TurnConformanceRunToolNotRegisteredError() throws Exception { + private static void vector123TurnConformanceRunToolNotRegisteredError() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector93TurnConformanceRunToolResultMessageFormat() throws Exception { + private static void vector124TurnConformanceRunToolResultMessageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\\u00b0F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\\u00b0F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\\u00b0F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\\u00b0F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "run", vector, false, seam()); } - private static void vector94TurnConformanceRunTurnCancelBeforeContext() throws Exception { + private static void vector125TurnConformanceRunTurnCancelBeforeContext() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector95TurnConformanceRunTurnDelegatedProviderState() throws Exception { + private static void vector126TurnConformanceRunTurnDelegatedProviderState() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector96TurnConformanceRunTurnFinalOutput() throws Exception { + private static void vector127TurnConformanceRunTurnFinalOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector97TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { + private static void vector128TurnConformanceRunTurnMemoryContextIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector98TurnConformanceRunTurnOrderedToolRound() throws Exception { + private static void vector129TurnConformanceRunTurnOrderedToolRound() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector99TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { + private static void vector130TurnConformanceRunTurnPermissionDenialIsModelVisible() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("TurnConformance", "runTurn", vector, false, seam()); } - private static void vector100WireConformanceToRequestAnthropicImageFormat() throws Exception { + private static void vector131WireConformanceToRequestAnthropicImageFormat() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_image_format\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector101WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { + private static void vector132WireConformanceToRequestAnthropicMaxTokensRequired() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector102WireConformanceToRequestAnthropicOptions() throws Exception { + private static void vector133WireConformanceToRequestAnthropicOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector103WireConformanceToRequestAnthropicSystemSeparate() throws Exception { + private static void vector134WireConformanceToRequestAnthropicSystemSeparate() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_system_separate\",\"description\":\"\\u00a77.5 \\u2014 Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector104WireConformanceToRequestAnthropicToolWire() throws Exception { + private static void vector135WireConformanceToRequestAnthropicToolWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector105WireConformanceToRequestChatAudioMp3() throws Exception { + private static void vector136WireConformanceToRequestChatAudioMp3() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_mp3\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector106WireConformanceToRequestChatAudioPart() throws Exception { + private static void vector137WireConformanceToRequestChatAudioPart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_audio_part\",\"description\":\"\\u00a77.1.2 \\u2014 AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector107WireConformanceToRequestChatImageBase64() throws Exception { + private static void vector138WireConformanceToRequestChatImageBase64() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector108WireConformanceToRequestChatImagePart() throws Exception { + private static void vector139WireConformanceToRequestChatImagePart() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector109WireConformanceToRequestChatMultipartContent() throws Exception { + private static void vector140WireConformanceToRequestChatMultipartContent() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_multipart_content\",\"description\":\"\\u00a77.1.1/\\u00a77.1.2 \\u2014 Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector110WireConformanceToRequestChatSimple() throws Exception { + private static void vector141WireConformanceToRequestChatSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_simple\",\"description\":\"\\u00a77.1.7 \\u2014 Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector111WireConformanceToRequestChatSingleTextOptimized() throws Exception { + private static void vector142WireConformanceToRequestChatSingleTextOptimized() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_single_text_optimized\",\"description\":\"\\u00a77.1.1 \\u2014 Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector112WireConformanceToRequestChatWithOptions() throws Exception { + private static void vector143WireConformanceToRequestChatWithOptions() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"chat_with_options\",\"description\":\"\\u00a77.1.5 \\u2014 All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector113WireConformanceToRequestEmbeddingWire() throws Exception { + private static void vector144WireConformanceToRequestEmbeddingWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"embedding_wire\",\"description\":\"\\u00a77.2 \\u2014 Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector114WireConformanceToRequestImageWire() throws Exception { + private static void vector145WireConformanceToRequestImageWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"image_wire\",\"description\":\"\\u00a77.3 \\u2014 Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector115WireConformanceToRequestKindToJsonTypeMapping() throws Exception { + private static void vector146WireConformanceToRequestKindToJsonTypeMapping() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\\u00a77.1.4 \\u2014 All Property kind values mapped to JSON Schema types: string\\u2192string, integer\\u2192integer, float\\u2192number, boolean\\u2192boolean, array\\u2192array, object\\u2192object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector116WireConformanceToRequestOptionsAdditionalProperties() throws Exception { + private static void vector147WireConformanceToRequestOptionsAdditionalProperties() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_additional_properties\",\"description\":\"\\u00a77.1.5 \\u2014 additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector117WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { + private static void vector148WireConformanceToRequestOptionsMaxCompletionTokens() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_max_completion_tokens\",\"description\":\"\\u00a77.1.5 \\u2014 maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector118WireConformanceToRequestOptionsStopSequences() throws Exception { + private static void vector149WireConformanceToRequestOptionsStopSequences() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"options_stop_sequences\",\"description\":\"\\u00a77.1.5 \\u2014 stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector119WireConformanceToRequestResponsesSimple() throws Exception { + private static void vector150WireConformanceToRequestResponsesSimple() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector120WireConformanceToRequestResponsesStructuredOutput() throws Exception { + private static void vector151WireConformanceToRequestResponsesStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector121WireConformanceToRequestResponsesWithTools() throws Exception { + private static void vector152WireConformanceToRequestResponsesWithTools() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector122WireConformanceToRequestStructuredOutput() throws Exception { + private static void vector153WireConformanceToRequestStructuredOutput() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output\",\"description\":\"\\u00a77.1.6 \\u2014 outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector123WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { + private static void vector154WireConformanceToRequestStructuredOutputNestedOptional() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"structured_output_nested_optional\",\"description\":\"\\u00a77.1.4/\\u00a77.1.6 \\u2014 OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector124WireConformanceToRequestToolsBindingsStripped() throws Exception { + private static void vector155WireConformanceToRequestToolsBindingsStripped() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_bindings_stripped\",\"description\":\"\\u00a77.1.3 \\u2014 Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector125WireConformanceToRequestToolsFunctionWire() throws Exception { + private static void vector156WireConformanceToRequestToolsFunctionWire() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_function_wire\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector126WireConformanceToRequestToolsNullWhenEmpty() throws Exception { + private static void vector157WireConformanceToRequestToolsNullWhenEmpty() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_null_when_empty\",\"description\":\"\\u00a77.1.3 \\u2014 When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); VectorRunner.runVector("WireConformance", "toRequest", vector, false, seam()); } - private static void vector127WireConformanceToRequestToolsStrictMode() throws Exception { + private static void vector158WireConformanceToRequestToolsStrictMode() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("{\"name\":\"tools_strict_mode\",\"description\":\"\\u00a77.1.3 \\u2014 FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}"); Object vector = TypraJson.parse(sb.toString()); @@ -1003,86 +1220,117 @@ public static void run() { failed += runCase("LoadConformance.load:tools_function_load", VectorConformanceTests::vector45LoadConformanceLoadToolsFunctionLoad); failed += runCase("LoadConformance.load:tools_mcp_load", VectorConformanceTests::vector46LoadConformanceLoadToolsMcpLoad); failed += runCase("LoadConformance.load:tools_openapi_load", VectorConformanceTests::vector47LoadConformanceLoadToolsOpenapiLoad); - failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector48MemoryConformanceOperateClearByCategoryAndAll); - failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector49MemoryConformanceOperateFormatCoreAndRecallResults); - failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector50MemoryConformanceOperateRecallEmptyQueryAndLimit); - failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector51MemoryConformanceOperateRecallRanksByWeightedScore); - failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); - failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector53MemoryConformanceOperateRecallTagsAndCoreBoost); - failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); - failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector55MemoryConformanceOperateRememberDedupsCoreByTags); - failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); - failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector57MemoryConformanceOperateRememberEvictsArchivalFirst); - failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector58MemoryConformanceOperateSnapshotPortRoundTrip); - failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector59MemoryConformanceOperateUpdateIndexOutOfRangeError); - failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector60TurnConformanceReplayMaxIterations); - failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector61TurnConformanceReplayNoTool); - failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector62TurnConformanceReplayPermissionDenied); - failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector63TurnConformanceReplayToolFailure); - failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector64TurnConformanceReplayToolSuccess); - failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector65TurnConformanceRunAssistantToolCallsMetadata); - failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector66TurnConformanceRunAsyncToolFunction); - failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector67TurnConformanceRunBindingsInjected); - failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector68TurnConformanceRunCancellationBeforeLlm); - failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector69TurnConformanceRunCancellationBetweenIterations); - failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector70TurnConformanceRunCancellationBetweenTools); - failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector71TurnConformanceRunContextNoTrimWhenFits); - failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector72TurnConformanceRunContextPreservesSystemMessages); - failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector73TurnConformanceRunContextTrimBasic); - failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector74TurnConformanceRunEmptyToolResult); - failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector75TurnConformanceRunEventsBasicToolLoop); - failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector76TurnConformanceRunEventsErrorLogged); - failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector77TurnConformanceRunEventsNoTools); - failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector78TurnConformanceRunGuardrailAllPass); - failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector79TurnConformanceRunGuardrailInputDeny); - failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector80TurnConformanceRunGuardrailOutputDeny); - failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector81TurnConformanceRunGuardrailToolDeny); - failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector82TurnConformanceRunMaxIterationsExceeded); - failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector83TurnConformanceRunMultiTurnToolCalls); - failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector84TurnConformanceRunMultipleToolCallsSingleTurn); - failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector85TurnConformanceRunNoToolCalls); - failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector86TurnConformanceRunParallelToolsBasic); - failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector87TurnConformanceRunParallelToolsWithGuardrailDeny); - failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector88TurnConformanceRunParallelToolsWithToolError); - failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector89TurnConformanceRunSingleToolCall); - failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector90TurnConformanceRunSteeringInjectMessage); - failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector91TurnConformanceRunSteeringMultipleMessages); - failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector92TurnConformanceRunToolNotRegisteredError); - failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector93TurnConformanceRunToolResultMessageFormat); - failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector94TurnConformanceRunTurnCancelBeforeContext); - failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector95TurnConformanceRunTurnDelegatedProviderState); - failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector96TurnConformanceRunTurnFinalOutput); - failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector97TurnConformanceRunTurnMemoryContextIsModelVisible); - failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector98TurnConformanceRunTurnOrderedToolRound); - failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector99TurnConformanceRunTurnPermissionDenialIsModelVisible); - failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector100WireConformanceToRequestAnthropicImageFormat); - failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector101WireConformanceToRequestAnthropicMaxTokensRequired); - failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector102WireConformanceToRequestAnthropicOptions); - failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector103WireConformanceToRequestAnthropicSystemSeparate); - failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector104WireConformanceToRequestAnthropicToolWire); - failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector105WireConformanceToRequestChatAudioMp3); - failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector106WireConformanceToRequestChatAudioPart); - failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector107WireConformanceToRequestChatImageBase64); - failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector108WireConformanceToRequestChatImagePart); - failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector109WireConformanceToRequestChatMultipartContent); - failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector110WireConformanceToRequestChatSimple); - failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector111WireConformanceToRequestChatSingleTextOptimized); - failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector112WireConformanceToRequestChatWithOptions); - failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector113WireConformanceToRequestEmbeddingWire); - failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector114WireConformanceToRequestImageWire); - failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector115WireConformanceToRequestKindToJsonTypeMapping); - failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector116WireConformanceToRequestOptionsAdditionalProperties); - failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector117WireConformanceToRequestOptionsMaxCompletionTokens); - failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector118WireConformanceToRequestOptionsStopSequences); - failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector119WireConformanceToRequestResponsesSimple); - failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector120WireConformanceToRequestResponsesStructuredOutput); - failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector121WireConformanceToRequestResponsesWithTools); - failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector122WireConformanceToRequestStructuredOutput); - failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector123WireConformanceToRequestStructuredOutputNestedOptional); - failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector124WireConformanceToRequestToolsBindingsStripped); - failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector125WireConformanceToRequestToolsFunctionWire); - failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector126WireConformanceToRequestToolsNullWhenEmpty); - failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector127WireConformanceToRequestToolsStrictMode); + failed += runCase("LoadConformance.load:trusted_endpoint_builtin_allowed", VectorConformanceTests::vector48LoadConformanceLoadTrustedEndpointBuiltinAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_default_port_allowed", VectorConformanceTests::vector49LoadConformanceLoadTrustedEndpointDefaultPortAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_env_endpoint_allowed", VectorConformanceTests::vector50LoadConformanceLoadTrustedEndpointEnvEndpointAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_env_list_allowed", VectorConformanceTests::vector51LoadConformanceLoadTrustedEndpointEnvListAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_literal_key_allowed", VectorConformanceTests::vector52LoadConformanceLoadTrustedEndpointLiteralKeyAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_option_allowed", VectorConformanceTests::vector53LoadConformanceLoadTrustedEndpointOptionAllowed); + failed += runCase("LoadConformance.load:trusted_endpoint_reference_skipped", VectorConformanceTests::vector54LoadConformanceLoadTrustedEndpointReferenceSkipped); + failed += runCase("LoadConformance.load:trusted_endpoint_userinfo_allowed", VectorConformanceTests::vector55LoadConformanceLoadTrustedEndpointUserinfoAllowed); + failed += runCase("LoadConformance.load:untrusted_endpoint_ambient_rejected", VectorConformanceTests::vector56LoadConformanceLoadUntrustedEndpointAmbientRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_backslash_rejected", VectorConformanceTests::vector57LoadConformanceLoadUntrustedEndpointBackslashRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_blank_literal_key_rejected", VectorConformanceTests::vector58LoadConformanceLoadUntrustedEndpointBlankLiteralKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_env_default_rejected", VectorConformanceTests::vector59LoadConformanceLoadUntrustedEndpointEnvDefaultRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_env_key_rejected", VectorConformanceTests::vector60LoadConformanceLoadUntrustedEndpointEnvKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_file_connection_rejected", VectorConformanceTests::vector61LoadConformanceLoadUntrustedEndpointFileConnectionRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_fragment_before_at_rejected", VectorConformanceTests::vector62LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_ipv6_literal_rejected", VectorConformanceTests::vector63LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", VectorConformanceTests::vector64LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", VectorConformanceTests::vector65LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_no_key_rejected", VectorConformanceTests::vector66LoadConformanceLoadUntrustedEndpointNoKeyRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_oauth_literal_secret_rejected", VectorConformanceTests::vector67LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_percent_encoded_host_rejected", VectorConformanceTests::vector68LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_port_mismatch_rejected", VectorConformanceTests::vector69LoadConformanceLoadUntrustedEndpointPortMismatchRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_query_before_at_rejected", VectorConformanceTests::vector70LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_scheme_mismatch_rejected", VectorConformanceTests::vector71LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_lookalike_rejected", VectorConformanceTests::vector72LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", VectorConformanceTests::vector73LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_trailing_dot_rejected", VectorConformanceTests::vector74LoadConformanceLoadUntrustedEndpointTrailingDotRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_unicode_homoglyph_rejected", VectorConformanceTests::vector75LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_userinfo_lookalike_rejected", VectorConformanceTests::vector76LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", VectorConformanceTests::vector77LoadConformanceLoadUntrustedEndpointWildcardApexRejected); + failed += runCase("LoadConformance.load:untrusted_endpoint_wildcard_boundary_rejected", VectorConformanceTests::vector78LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected); + failed += runCase("MemoryConformance.operate:clear_by_category_and_all", VectorConformanceTests::vector79MemoryConformanceOperateClearByCategoryAndAll); + failed += runCase("MemoryConformance.operate:format_core_and_recall_results", VectorConformanceTests::vector80MemoryConformanceOperateFormatCoreAndRecallResults); + failed += runCase("MemoryConformance.operate:recall_empty_query_and_limit", VectorConformanceTests::vector81MemoryConformanceOperateRecallEmptyQueryAndLimit); + failed += runCase("MemoryConformance.operate:recall_ranks_by_weighted_score", VectorConformanceTests::vector82MemoryConformanceOperateRecallRanksByWeightedScore); + failed += runCase("MemoryConformance.operate:recall_score_ties_preserve_insertion_order", VectorConformanceTests::vector83MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder); + failed += runCase("MemoryConformance.operate:recall_tags_and_core_boost", VectorConformanceTests::vector84MemoryConformanceOperateRecallTagsAndCoreBoost); + failed += runCase("MemoryConformance.operate:recall_tokenization_punctuation_unicode_and_dedup", VectorConformanceTests::vector85MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup); + failed += runCase("MemoryConformance.operate:remember_dedups_core_by_tags", VectorConformanceTests::vector86MemoryConformanceOperateRememberDedupsCoreByTags); + failed += runCase("MemoryConformance.operate:remember_dedups_core_missing_and_empty_tags", VectorConformanceTests::vector87MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags); + failed += runCase("MemoryConformance.operate:remember_evicts_archival_first", VectorConformanceTests::vector88MemoryConformanceOperateRememberEvictsArchivalFirst); + failed += runCase("MemoryConformance.operate:snapshot_port_round_trip", VectorConformanceTests::vector89MemoryConformanceOperateSnapshotPortRoundTrip); + failed += runCase("MemoryConformance.operate:update_index_out_of_range_error", VectorConformanceTests::vector90MemoryConformanceOperateUpdateIndexOutOfRangeError); + failed += runCase("TurnConformance.replay:max_iterations", VectorConformanceTests::vector91TurnConformanceReplayMaxIterations); + failed += runCase("TurnConformance.replay:no_tool", VectorConformanceTests::vector92TurnConformanceReplayNoTool); + failed += runCase("TurnConformance.replay:permission_denied", VectorConformanceTests::vector93TurnConformanceReplayPermissionDenied); + failed += runCase("TurnConformance.replay:tool_failure", VectorConformanceTests::vector94TurnConformanceReplayToolFailure); + failed += runCase("TurnConformance.replay:tool_success", VectorConformanceTests::vector95TurnConformanceReplayToolSuccess); + failed += runCase("TurnConformance.run:assistant_tool_calls_metadata", VectorConformanceTests::vector96TurnConformanceRunAssistantToolCallsMetadata); + failed += runCase("TurnConformance.run:async_tool_function", VectorConformanceTests::vector97TurnConformanceRunAsyncToolFunction); + failed += runCase("TurnConformance.run:bindings_injected", VectorConformanceTests::vector98TurnConformanceRunBindingsInjected); + failed += runCase("TurnConformance.run:cancellation_before_llm", VectorConformanceTests::vector99TurnConformanceRunCancellationBeforeLlm); + failed += runCase("TurnConformance.run:cancellation_between_iterations", VectorConformanceTests::vector100TurnConformanceRunCancellationBetweenIterations); + failed += runCase("TurnConformance.run:cancellation_between_tools", VectorConformanceTests::vector101TurnConformanceRunCancellationBetweenTools); + failed += runCase("TurnConformance.run:context_no_trim_when_fits", VectorConformanceTests::vector102TurnConformanceRunContextNoTrimWhenFits); + failed += runCase("TurnConformance.run:context_preserves_system_messages", VectorConformanceTests::vector103TurnConformanceRunContextPreservesSystemMessages); + failed += runCase("TurnConformance.run:context_trim_basic", VectorConformanceTests::vector104TurnConformanceRunContextTrimBasic); + failed += runCase("TurnConformance.run:empty_tool_result", VectorConformanceTests::vector105TurnConformanceRunEmptyToolResult); + failed += runCase("TurnConformance.run:events_basic_tool_loop", VectorConformanceTests::vector106TurnConformanceRunEventsBasicToolLoop); + failed += runCase("TurnConformance.run:events_error_logged", VectorConformanceTests::vector107TurnConformanceRunEventsErrorLogged); + failed += runCase("TurnConformance.run:events_no_tools", VectorConformanceTests::vector108TurnConformanceRunEventsNoTools); + failed += runCase("TurnConformance.run:guardrail_all_pass", VectorConformanceTests::vector109TurnConformanceRunGuardrailAllPass); + failed += runCase("TurnConformance.run:guardrail_input_deny", VectorConformanceTests::vector110TurnConformanceRunGuardrailInputDeny); + failed += runCase("TurnConformance.run:guardrail_output_deny", VectorConformanceTests::vector111TurnConformanceRunGuardrailOutputDeny); + failed += runCase("TurnConformance.run:guardrail_tool_deny", VectorConformanceTests::vector112TurnConformanceRunGuardrailToolDeny); + failed += runCase("TurnConformance.run:max_iterations_exceeded", VectorConformanceTests::vector113TurnConformanceRunMaxIterationsExceeded); + failed += runCase("TurnConformance.run:multi_turn_tool_calls", VectorConformanceTests::vector114TurnConformanceRunMultiTurnToolCalls); + failed += runCase("TurnConformance.run:multiple_tool_calls_single_turn", VectorConformanceTests::vector115TurnConformanceRunMultipleToolCallsSingleTurn); + failed += runCase("TurnConformance.run:no_tool_calls", VectorConformanceTests::vector116TurnConformanceRunNoToolCalls); + failed += runCase("TurnConformance.run:parallel_tools_basic", VectorConformanceTests::vector117TurnConformanceRunParallelToolsBasic); + failed += runCase("TurnConformance.run:parallel_tools_with_guardrail_deny", VectorConformanceTests::vector118TurnConformanceRunParallelToolsWithGuardrailDeny); + failed += runCase("TurnConformance.run:parallel_tools_with_tool_error", VectorConformanceTests::vector119TurnConformanceRunParallelToolsWithToolError); + failed += runCase("TurnConformance.run:single_tool_call", VectorConformanceTests::vector120TurnConformanceRunSingleToolCall); + failed += runCase("TurnConformance.run:steering_inject_message", VectorConformanceTests::vector121TurnConformanceRunSteeringInjectMessage); + failed += runCase("TurnConformance.run:steering_multiple_messages", VectorConformanceTests::vector122TurnConformanceRunSteeringMultipleMessages); + failed += runCase("TurnConformance.run:tool_not_registered_error", VectorConformanceTests::vector123TurnConformanceRunToolNotRegisteredError); + failed += runCase("TurnConformance.run:tool_result_message_format", VectorConformanceTests::vector124TurnConformanceRunToolResultMessageFormat); + failed += runCase("TurnConformance.runTurn:cancel_before_context", VectorConformanceTests::vector125TurnConformanceRunTurnCancelBeforeContext); + failed += runCase("TurnConformance.runTurn:delegated_provider_state", VectorConformanceTests::vector126TurnConformanceRunTurnDelegatedProviderState); + failed += runCase("TurnConformance.runTurn:final_output", VectorConformanceTests::vector127TurnConformanceRunTurnFinalOutput); + failed += runCase("TurnConformance.runTurn:memory_context_is_model_visible", VectorConformanceTests::vector128TurnConformanceRunTurnMemoryContextIsModelVisible); + failed += runCase("TurnConformance.runTurn:ordered_tool_round", VectorConformanceTests::vector129TurnConformanceRunTurnOrderedToolRound); + failed += runCase("TurnConformance.runTurn:permission_denial_is_model_visible", VectorConformanceTests::vector130TurnConformanceRunTurnPermissionDenialIsModelVisible); + failed += runCase("WireConformance.toRequest:anthropic_image_format", VectorConformanceTests::vector131WireConformanceToRequestAnthropicImageFormat); + failed += runCase("WireConformance.toRequest:anthropic_max_tokens_required", VectorConformanceTests::vector132WireConformanceToRequestAnthropicMaxTokensRequired); + failed += runCase("WireConformance.toRequest:anthropic_options", VectorConformanceTests::vector133WireConformanceToRequestAnthropicOptions); + failed += runCase("WireConformance.toRequest:anthropic_system_separate", VectorConformanceTests::vector134WireConformanceToRequestAnthropicSystemSeparate); + failed += runCase("WireConformance.toRequest:anthropic_tool_wire", VectorConformanceTests::vector135WireConformanceToRequestAnthropicToolWire); + failed += runCase("WireConformance.toRequest:chat_audio_mp3", VectorConformanceTests::vector136WireConformanceToRequestChatAudioMp3); + failed += runCase("WireConformance.toRequest:chat_audio_part", VectorConformanceTests::vector137WireConformanceToRequestChatAudioPart); + failed += runCase("WireConformance.toRequest:chat_image_base64", VectorConformanceTests::vector138WireConformanceToRequestChatImageBase64); + failed += runCase("WireConformance.toRequest:chat_image_part", VectorConformanceTests::vector139WireConformanceToRequestChatImagePart); + failed += runCase("WireConformance.toRequest:chat_multipart_content", VectorConformanceTests::vector140WireConformanceToRequestChatMultipartContent); + failed += runCase("WireConformance.toRequest:chat_simple", VectorConformanceTests::vector141WireConformanceToRequestChatSimple); + failed += runCase("WireConformance.toRequest:chat_single_text_optimized", VectorConformanceTests::vector142WireConformanceToRequestChatSingleTextOptimized); + failed += runCase("WireConformance.toRequest:chat_with_options", VectorConformanceTests::vector143WireConformanceToRequestChatWithOptions); + failed += runCase("WireConformance.toRequest:embedding_wire", VectorConformanceTests::vector144WireConformanceToRequestEmbeddingWire); + failed += runCase("WireConformance.toRequest:image_wire", VectorConformanceTests::vector145WireConformanceToRequestImageWire); + failed += runCase("WireConformance.toRequest:kind_to_json_type_mapping", VectorConformanceTests::vector146WireConformanceToRequestKindToJsonTypeMapping); + failed += runCase("WireConformance.toRequest:options_additional_properties", VectorConformanceTests::vector147WireConformanceToRequestOptionsAdditionalProperties); + failed += runCase("WireConformance.toRequest:options_max_completion_tokens", VectorConformanceTests::vector148WireConformanceToRequestOptionsMaxCompletionTokens); + failed += runCase("WireConformance.toRequest:options_stop_sequences", VectorConformanceTests::vector149WireConformanceToRequestOptionsStopSequences); + failed += runCase("WireConformance.toRequest:responses_simple", VectorConformanceTests::vector150WireConformanceToRequestResponsesSimple); + failed += runCase("WireConformance.toRequest:responses_structured_output", VectorConformanceTests::vector151WireConformanceToRequestResponsesStructuredOutput); + failed += runCase("WireConformance.toRequest:responses_with_tools", VectorConformanceTests::vector152WireConformanceToRequestResponsesWithTools); + failed += runCase("WireConformance.toRequest:structured_output", VectorConformanceTests::vector153WireConformanceToRequestStructuredOutput); + failed += runCase("WireConformance.toRequest:structured_output_nested_optional", VectorConformanceTests::vector154WireConformanceToRequestStructuredOutputNestedOptional); + failed += runCase("WireConformance.toRequest:tools_bindings_stripped", VectorConformanceTests::vector155WireConformanceToRequestToolsBindingsStripped); + failed += runCase("WireConformance.toRequest:tools_function_wire", VectorConformanceTests::vector156WireConformanceToRequestToolsFunctionWire); + failed += runCase("WireConformance.toRequest:tools_null_when_empty", VectorConformanceTests::vector157WireConformanceToRequestToolsNullWhenEmpty); + failed += runCase("WireConformance.toRequest:tools_strict_mode", VectorConformanceTests::vector158WireConformanceToRequestToolsStrictMode); if (failed > 0) { throw new AssertionError(failed + " @vector conformance failure(s)"); } diff --git a/runtime/python/prompty/README.md b/runtime/python/prompty/README.md index 8c0317943..2dc20f5c5 100644 --- a/runtime/python/prompty/README.md +++ b/runtime/python/prompty/README.md @@ -294,6 +294,14 @@ input value. `${file:...}` references are scoped to the containing `.prompty` file's directory by default. Pass `allowed_file_roots` to `load()` / `load_async()` only when the host application explicitly trusts additional shared directories. +### Endpoint Trust + +A `.prompty` file can't send ambient credentials (`DefaultAzureCredential` tokens, `OPENAI_API_KEY` and similar env keys) to an endpoint it picked itself. Unless the connection uses a literal `apiKey` (`kind: key`), a literal `endpoint`/`tokenUrl` must be trusted, or `load()` raises `PromptyLoadError("untrusted_endpoint")`. + +- Built-in trusted origins: `api.openai.com`, `api.anthropic.com`, `*.openai.azure.com`, `*.services.ai.azure.com`, `*.cognitiveservices.azure.com`, `login.microsoftonline.com` (all `https`). +- Supply the endpoint from the host: `endpoint: ${env:MY_ENDPOINT}`. +- Or trust more origins: `load(path, trusted_endpoints=["http://localhost:11434"])` or `PROMPTY_TRUSTED_ENDPOINTS=http://localhost:11434,https://*.contoso.com`. + ### Connection Kinds | Kind | When to use | diff --git a/runtime/python/prompty/prompty/core/endpoint_trust.py b/runtime/python/prompty/prompty/core/endpoint_trust.py new file mode 100644 index 000000000..55f962575 --- /dev/null +++ b/runtime/python/prompty/prompty/core/endpoint_trust.py @@ -0,0 +1,129 @@ +"""Endpoint trust — keep ambient credentials on host-approved destinations (spec §2.5.1). + +A ``.prompty`` file is untrusted input. When its connection carries no +file-supplied credential, executors fall back to ambient credentials +(``DefaultAzureCredential``, ``*_API_KEY`` env vars). This module decides +whether the file-chosen destination may receive them. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterable, Mapping +from typing import Any +from urllib.parse import urlsplit + +from .errors import PromptyLoadError + +__all__ = ["BUILTIN_TRUSTED_ENDPOINTS", "TRUSTED_ENDPOINTS_ENV", "check_connection_trust", "is_trusted_endpoint"] + +TRUSTED_ENDPOINTS_ENV = "PROMPTY_TRUSTED_ENDPOINTS" + +BUILTIN_TRUSTED_ENDPOINTS: tuple[str, ...] = ( + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com", +) + +_DEFAULT_PORTS = {"http": 80, "https": 443} +_CREDENTIAL_KEYS: dict[str, str] = {"key": "apiKey"} +_DESTINATION_KEYS = ("endpoint", "tokenUrl") + + +def _origin(url: str) -> tuple[str, str, int] | None: + """Return ``(scheme, host, port)`` for an absolute http(s) URL, else ``None``.""" + # HTTP clients treat '\' as '/', so parsers disagree on the host; reject outright. + if "\\" in url: + return None + try: + parts = urlsplit(url.strip()) + port = parts.port + except ValueError: + return None + scheme = parts.scheme.lower() + host = (parts.hostname or "").lower() + if scheme not in _DEFAULT_PORTS or not host: + return None + return scheme, host, port if port is not None else _DEFAULT_PORTS[scheme] + + +def _trusted_entries(trusted_endpoints: Iterable[str] | None) -> list[str]: + entries = [*BUILTIN_TRUSTED_ENDPOINTS, *(trusted_endpoints or ())] + entries.extend(os.environ.get(TRUSTED_ENDPOINTS_ENV, "").split(",")) + return [e.strip() for e in entries if isinstance(e, str) and e.strip()] + + +def is_trusted_endpoint(url: str, trusted_endpoints: Iterable[str] | None = None) -> bool: + """Return ``True`` if ``url``'s origin matches a built-in, host, or env trusted origin.""" + entries = _trusted_entries(trusted_endpoints) + if "*" in entries: + return True + target = _origin(url) + if target is None: + return False + scheme, host, port = target + for entry in entries: + allowed = _origin(entry) + if allowed is None or allowed[0] != scheme or allowed[2] != port: + continue + pattern = allowed[1] + if pattern.startswith("*."): + if host.endswith(pattern[1:]): + return True + elif pattern == host: + return True + return False + + +def _is_reference(value: str) -> bool: + return value.startswith("${") and value.endswith("}") + + +def _file_supplied_credential(connection: Mapping[str, Any]) -> bool: + # Only the credential field the declared kind actually consumes counts; any other kind is ambient. + kind = connection.get("kind") + key = _CREDENTIAL_KEYS.get(kind) if isinstance(kind, str) else None + if key is None: + return False + value = connection.get(key) + # Blank keys count as absent: executors that trim fall back to ambient credentials. + return isinstance(value, str) and bool(value.strip()) and not _is_reference(value) + + +def _file_controlled_destination(value: Any) -> str | None: + """Return the effective destination if the file chose it, else ``None``.""" + if value is None: + return None + if not isinstance(value, str): + return str(value) + if _is_reference(value): + protocol, _, rest = value[2:-1].partition(":") + if protocol.lower() == "env": + var_name, _, default = rest.partition(":") + if os.environ.get(var_name) is not None: + return None + return default or None + return value + + +def check_connection_trust(connection: Any, trusted_endpoints: Iterable[str] | None = None) -> None: + """Raise ``untrusted_endpoint`` if ambient credentials could reach a file-chosen, untrusted destination. + + ``connection`` is the RAW (unresolved) ``model.connection`` mapping. + """ + if not isinstance(connection, Mapping) or connection.get("kind") == "reference": + return + if _file_supplied_credential(connection): + return + for key in _DESTINATION_KEYS: + destination = _file_controlled_destination(connection.get(key)) + if destination is not None and not is_trusted_endpoint(destination, trusted_endpoints): + raise PromptyLoadError( + "untrusted_endpoint", + f"Connection {key} '{destination}' is not trusted to receive ambient credentials. " + f"Set the {key} via ${{env:VAR}}, add its origin to trusted_endpoints or " + f"{TRUSTED_ENDPOINTS_ENV}, or use 'kind: reference'.", + ) diff --git a/runtime/python/prompty/prompty/core/loader.py b/runtime/python/prompty/prompty/core/loader.py index 64205eeaa..33588438f 100644 --- a/runtime/python/prompty/prompty/core/loader.py +++ b/runtime/python/prompty/prompty/core/loader.py @@ -16,6 +16,7 @@ import yaml from ..model import Agent, LoadContext, SaveContext +from .endpoint_trust import check_connection_trust from .errors import PromptyLoadError from .utils import load_prompty, load_prompty_async @@ -27,7 +28,12 @@ # --------------------------------------------------------------------------- -def load(path: str | Path, *, allowed_file_roots: Sequence[str | Path] | None = None) -> Agent: +def load( + path: str | Path, + *, + allowed_file_roots: Sequence[str | Path] | None = None, + trusted_endpoints: Sequence[str] | None = None, +) -> Agent: """Load a ``.prompty`` file and return a typed ``Agent``. Parameters @@ -38,6 +44,10 @@ def load(path: str | Path, *, allowed_file_roots: Sequence[str | Path] | None = Additional directories that ``${file:...}`` references may read from. The prompt file's directory is always allowed. References that resolve outside all allowed roots after canonicalization are rejected. + trusted_endpoints: + Additional origins (``scheme://host[:port]`` or ``scheme://*.domain``) that + may receive ambient credentials when the file supplies none. Merged with + built-in provider origins and ``PROMPTY_TRUSTED_ENDPOINTS``. Returns ------- @@ -52,17 +62,22 @@ def load(path: str | Path, *, allowed_file_roots: Sequence[str | Path] | None = data = load_prompty(path) # 2–7 shared pipeline - return _build_agent(data, path, allowed_file_roots=allowed_file_roots) + return _build_agent(data, path, allowed_file_roots=allowed_file_roots, trusted_endpoints=trusted_endpoints) -async def load_async(path: str | Path, *, allowed_file_roots: Sequence[str | Path] | None = None) -> Agent: +async def load_async( + path: str | Path, + *, + allowed_file_roots: Sequence[str | Path] | None = None, + trusted_endpoints: Sequence[str] | None = None, +) -> Agent: """Async variant of :func:`load`.""" path = Path(path).resolve() if not path.exists(): raise FileNotFoundError(f"Prompty file not found: {path}") data = await load_prompty_async(path) - return _build_agent(data, path, allowed_file_roots=allowed_file_roots) + return _build_agent(data, path, allowed_file_roots=allowed_file_roots, trusted_endpoints=trusted_endpoints) def default_save_context(**kwargs: Any) -> SaveContext: @@ -96,6 +111,7 @@ def _build_agent( path: Path, *, allowed_file_roots: Sequence[str | Path] | None = None, + trusted_endpoints: Sequence[str] | None = None, ) -> Agent: """Shared pipeline that transforms raw frontmatter dict into a Agent.""" @@ -116,6 +132,9 @@ def _build_agent( f"Invalid template format: {tmpl!r} (expected an object with format/parser)", ) + # Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. + check_connection_trust(_raw_connection(data, path, allowed_file_roots), trusted_endpoints) + # 2. Load via Agent.load() with pre_process for ${protocol:value} expansion ctx = LoadContext(pre_process=_pre_process(path, allowed_file_roots=allowed_file_roots)) agent = Agent.load(data, ctx) @@ -130,6 +149,25 @@ def _build_agent( return agent +def _raw_connection( + data: dict[str, Any], + path: Path, + allowed_file_roots: Sequence[str | Path] | None, +) -> Any: + """Return the unresolved ``model.connection``, following whole-value ``${file:}`` references.""" + + def follow(value: Any) -> Any: + if isinstance(value, str) and value.startswith("${") and value.endswith("}"): + protocol, _, ref = value[2:-1].partition(":") + if protocol.lower() == "file": + file_path = _resolve_file_reference(path, ref, allowed_file_roots) + return _load_file_content(file_path) if file_path.exists() else None + return value + + model = follow(data.get("model")) + return follow(model.get("connection")) if isinstance(model, dict) else None + + # --------------------------------------------------------------------------- # Reference resolution via pre_process # --------------------------------------------------------------------------- diff --git a/runtime/python/prompty/prompty/model/vector_conformance.py b/runtime/python/prompty/prompty/model/vector_conformance.py index 27e45e03b..0c1f7975b 100644 --- a/runtime/python/prompty/prompty/model/vector_conformance.py +++ b/runtime/python/prompty/prompty/model/vector_conformance.py @@ -459,3 +459,309 @@ def run_load_conformance_conformance(seam: LoadConformance) -> None: assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( "tools_openapi_load misrouted" ) + # vector: trusted_endpoint_builtin_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://myres.services.ai.azure.com/api/projects/p"}}}' + ) + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_builtin_allowed misrouted" + ) + # vector: trusted_endpoint_default_port_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"foundry","endpoint":"https://API.OpenAI.com:443/v1"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_default_port_allowed misrouted" + ) + # vector: trusted_endpoint_env_endpoint_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"foundry","endpoint":"https://host-chosen.example.com"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_env_endpoint_allowed misrouted" + ) + # vector: trusted_endpoint_env_list_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"key","endpoint":"http://localhost:11434/v1"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_env_list_allowed misrouted" + ) + # vector: trusted_endpoint_literal_key_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"key","endpoint":"https://anywhere.example.com"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_literal_key_allowed misrouted" + ) + # vector: trusted_endpoint_option_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://proxy.example.com:443/openai/v1"}}}' + ) + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_option_allowed misrouted" + ) + # vector: trusted_endpoint_reference_skipped + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"reference","name":"my-connection"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_reference_skipped misrouted" + ) + # vector: trusted_endpoint_userinfo_allowed + source = json.loads("null") + result = seam.load(source) + if inspect.isawaitable(result): + result = asyncio.run(result) + expected = json.loads('{"model":{"connection":{"kind":"foundry","endpoint":"https://user@api.openai.com/v1"}}}') + assert json.dumps(result.save(), sort_keys=True) == json.dumps(expected, sort_keys=True), ( + "trusted_endpoint_userinfo_allowed misrouted" + ) + # vector: untrusted_endpoint_ambient_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_ambient_rejected: expected an error" + # vector: untrusted_endpoint_backslash_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_backslash_rejected: expected an error" + # vector: untrusted_endpoint_blank_literal_key_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_blank_literal_key_rejected: expected an error" + # vector: untrusted_endpoint_env_default_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_env_default_rejected: expected an error" + # vector: untrusted_endpoint_env_key_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_env_key_rejected: expected an error" + # vector: untrusted_endpoint_file_connection_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_file_connection_rejected: expected an error" + # vector: untrusted_endpoint_fragment_before_at_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_fragment_before_at_rejected: expected an error" + # vector: untrusted_endpoint_ipv6_literal_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_ipv6_literal_rejected: expected an error" + # vector: untrusted_endpoint_key_with_client_secret_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_key_with_client_secret_rejected: expected an error" + # vector: untrusted_endpoint_mismatched_credential_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_mismatched_credential_rejected: expected an error" + # vector: untrusted_endpoint_no_key_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_no_key_rejected: expected an error" + # vector: untrusted_endpoint_oauth_literal_secret_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_oauth_literal_secret_rejected: expected an error" + # vector: untrusted_endpoint_percent_encoded_host_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_percent_encoded_host_rejected: expected an error" + # vector: untrusted_endpoint_port_mismatch_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_port_mismatch_rejected: expected an error" + # vector: untrusted_endpoint_query_before_at_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_query_before_at_rejected: expected an error" + # vector: untrusted_endpoint_scheme_mismatch_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_scheme_mismatch_rejected: expected an error" + # vector: untrusted_endpoint_suffix_lookalike_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_suffix_lookalike_rejected: expected an error" + # vector: untrusted_endpoint_suffix_spoof_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_suffix_spoof_rejected: expected an error" + # vector: untrusted_endpoint_trailing_dot_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_trailing_dot_rejected: expected an error" + # vector: untrusted_endpoint_unicode_homoglyph_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_unicode_homoglyph_rejected: expected an error" + # vector: untrusted_endpoint_userinfo_lookalike_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_userinfo_lookalike_rejected: expected an error" + # vector: untrusted_endpoint_wildcard_apex_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_wildcard_apex_rejected: expected an error" + # vector: untrusted_endpoint_wildcard_boundary_rejected + source = json.loads("null") + caught = None + try: + result = seam.load(source) + if inspect.isawaitable(result): + asyncio.run(result) + except Exception as error: # noqa: BLE001 + caught = error + assert caught is not None, "untrusted_endpoint_wildcard_boundary_rejected: expected an error" diff --git a/runtime/python/prompty/tests/model/test_vector_conformance.py b/runtime/python/prompty/tests/model/test_vector_conformance.py index d01cf8642..e2184ff02 100644 --- a/runtime/python/prompty/tests/model/test_vector_conformance.py +++ b/runtime/python/prompty/tests/model/test_vector_conformance.py @@ -785,7 +785,10 @@ async def test_vector_21_loadconformance_load_connection_types_load(): "endpoint": "https://localhost:8080" } } - } + }, + "trustedEndpoints": [ + "https://localhost:8080" + ] }, "expected": { "model": { @@ -882,7 +885,8 @@ async def test_vector_24_loadconformance_load_env_default(): "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "${env:MISSING_VAR:fallback_value}" + "endpoint": "${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }, @@ -1697,7 +1701,945 @@ async def test_vector_47_loadconformance_load_tools_openapi_load(): await run_vector("LoadConformance", "load", vector, False, _SEAM) -async def test_vector_48_memoryconformance_operate_clear_by_category_and_all(): +async def test_vector_48_loadconformance_load_trusted_endpoint_builtin_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_builtin_allowed", + "description": "Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-builtin", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_49_loadconformance_load_trusted_endpoint_default_port_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_default_port_allowed", + "description": "An explicit default port normalizes to the built-in origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-default-port-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_50_loadconformance_load_trusted_endpoint_env_endpoint_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_env_endpoint_allowed", + "description": "An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-endpoint", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:VECTOR_FOUNDRY_ENDPOINT}" + } + } + }, + "env": { + "VECTOR_FOUNDRY_ENDPOINT": "https://host-chosen.example.com" + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://host-chosen.example.com" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_51_loadconformance_load_trusted_endpoint_env_list_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_env_list_allowed", + "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-list", + "model": { + "id": "llama3", + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "env": { + "PROMPTY_TRUSTED_ENDPOINTS": "https://other.example.com, http://localhost:11434" + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_52_loadconformance_load_trusted_endpoint_literal_key_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_literal_key_allowed", + "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-literal-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com", + "apiKey": "file-supplied-key" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_53_loadconformance_load_trusted_endpoint_option_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_option_allowed", + "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-option", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "trustedEndpoints": [ + "HTTPS://PROXY.Example.com" + ] + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_54_loadconformance_load_trusted_endpoint_reference_skipped(): + vector_json = r""" + { + "name": "trusted_endpoint_reference_skipped", + "description": "kind: reference connections carry no endpoint and are not subject to the trust check", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-reference", + "model": { + "id": "gpt-4", + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_55_loadconformance_load_trusted_endpoint_userinfo_allowed(): + vector_json = r""" + { + "name": "trusted_endpoint_userinfo_allowed", + "description": "Userinfo is ignored; the destination host api.openai.com is trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-userinfo-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_56_loadconformance_load_untrusted_endpoint_ambient_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_ambient_rejected", + "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ambient", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com/api/projects/p" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_57_loadconformance_load_untrusted_endpoint_backslash_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_backslash_rejected", + "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-backslash", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com\\@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_58_loadconformance_load_untrusted_endpoint_blank_literal_key_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_blank_literal_key_rejected", + "description": "a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-blank-key", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": " " + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_59_loadconformance_load_untrusted_endpoint_env_default_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_env_default_rejected", + "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-default", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:NONEXISTENT:https://attacker.example.com}" + } + } + }, + "env": {} + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_60_loadconformance_load_untrusted_endpoint_env_key_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_env_key_rejected", + "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": "${env:VECTOR_HOST_KEY}" + } + } + }, + "env": { + "VECTOR_HOST_KEY": "host-secret" + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_61_loadconformance_load_untrusted_endpoint_file_connection_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_file_connection_rejected", + "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-file-conn", + "model": { + "id": "gpt-4", + "connection": "${file:conn.json}" + } + }, + "files": { + "conn.json": { + "kind": "foundry", + "endpoint": "https://attacker.example.com" + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_62_loadconformance_load_untrusted_endpoint_fragment_before_at_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_fragment_before_at_rejected", + "description": "A '#' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-fragment-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com#@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_63_loadconformance_load_untrusted_endpoint_ipv6_literal_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_ipv6_literal_rejected", + "description": "An IPv6 literal is not a trusted origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ipv6-literal", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://[::1]:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_64_loadconformance_load_untrusted_endpoint_key_with_client_secret_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_key_with_client_secret_rejected", + "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-key-client-secret", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "clientSecret": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_65_loadconformance_load_untrusted_endpoint_mismatched_credential_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_mismatched_credential_rejected", + "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-mismatched-credential", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com", + "apiKey": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_66_loadconformance_load_untrusted_endpoint_no_key_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_no_key_rejected", + "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-no-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "http://127.0.0.1:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_67_loadconformance_load_untrusted_endpoint_oauth_literal_secret_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_oauth_literal_secret_rejected", + "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-oauth-literal", + "model": { + "id": "gpt-4", + "provider": "anthropic", + "connection": { + "kind": "oauth", + "endpoint": "https://attacker.example.com", + "clientId": "decoy", + "clientSecret": "decoy", + "tokenUrl": "https://login.microsoftonline.com/t/oauth2/v2.0/token" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_68_loadconformance_load_untrusted_endpoint_percent_encoded_host_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_percent_encoded_host_rejected", + "description": "Hosts are compared without percent-decoding, so an encoded host never matches", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-percent-encoded-host", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com%2eevil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_69_loadconformance_load_untrusted_endpoint_port_mismatch_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_port_mismatch_rejected", + "description": "A non-default port is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-port-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com:8443" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_70_loadconformance_load_untrusted_endpoint_query_before_at_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_query_before_at_rejected", + "description": "A '?' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-query-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com?@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_71_loadconformance_load_untrusted_endpoint_scheme_mismatch_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_scheme_mismatch_rejected", + "description": "Built-in origins are https; http on the same host is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-scheme-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_72_loadconformance_load_untrusted_endpoint_suffix_lookalike_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_suffix_lookalike_rejected", + "description": "A trusted host as a label prefix of another domain is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-suffix-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com.evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_73_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_suffix_spoof_rejected", + "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-spoof", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://good.example.com.attacker.example.com" + } + } + }, + "trustedEndpoints": [ + "https://good.example.com" + ] + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_74_loadconformance_load_untrusted_endpoint_trailing_dot_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_trailing_dot_rejected", + "description": "Hosts are compared exactly; a trailing-dot host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-trailing-dot", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com." + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_75_loadconformance_load_untrusted_endpoint_unicode_homoglyph_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_unicode_homoglyph_rejected", + "description": "A non-ASCII lookalike host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-unicode-homoglyph", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://аpi.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_76_loadconformance_load_untrusted_endpoint_userinfo_lookalike_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_userinfo_lookalike_rejected", + "description": "A trusted host in userinfo is not the destination host", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-userinfo-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com@evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_77_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_wildcard_apex_rejected", + "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-apex", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://myres.openai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_78_loadconformance_load_untrusted_endpoint_wildcard_boundary_rejected(): + vector_json = r""" + { + "name": "untrusted_endpoint_wildcard_boundary_rejected", + "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-wildcard-boundary", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evilservices.ai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + """ + vector = json.loads(vector_json, strict=False) + await run_vector("LoadConformance", "load", vector, False, _SEAM) + + +async def test_vector_79_memoryconformance_operate_clear_by_category_and_all(): vector_json = r""" { "name": "clear_by_category_and_all", @@ -1749,7 +2691,7 @@ async def test_vector_48_memoryconformance_operate_clear_by_category_and_all(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_49_memoryconformance_operate_format_core_and_recall_results(): +async def test_vector_80_memoryconformance_operate_format_core_and_recall_results(): vector_json = r""" { "name": "format_core_and_recall_results", @@ -1789,7 +2731,7 @@ async def test_vector_49_memoryconformance_operate_format_core_and_recall_result await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_50_memoryconformance_operate_recall_empty_query_and_limit(): +async def test_vector_81_memoryconformance_operate_recall_empty_query_and_limit(): vector_json = r""" { "name": "recall_empty_query_and_limit", @@ -1842,7 +2784,7 @@ async def test_vector_50_memoryconformance_operate_recall_empty_query_and_limit( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_51_memoryconformance_operate_recall_ranks_by_weighted_score(): +async def test_vector_82_memoryconformance_operate_recall_ranks_by_weighted_score(): vector_json = r""" { "name": "recall_ranks_by_weighted_score", @@ -1899,7 +2841,7 @@ async def test_vector_51_memoryconformance_operate_recall_ranks_by_weighted_scor await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_52_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): +async def test_vector_83_memoryconformance_operate_recall_score_ties_preserve_insertion_order(): vector_json = r""" { "name": "recall_score_ties_preserve_insertion_order", @@ -1955,7 +2897,7 @@ async def test_vector_52_memoryconformance_operate_recall_score_ties_preserve_in await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_53_memoryconformance_operate_recall_tags_and_core_boost(): +async def test_vector_84_memoryconformance_operate_recall_tags_and_core_boost(): vector_json = r""" { "name": "recall_tags_and_core_boost", @@ -2014,7 +2956,7 @@ async def test_vector_53_memoryconformance_operate_recall_tags_and_core_boost(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_54_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): +async def test_vector_85_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup(): vector_json = r""" { "name": "recall_tokenization_punctuation_unicode_and_dedup", @@ -2073,7 +3015,7 @@ async def test_vector_54_memoryconformance_operate_recall_tokenization_punctuati await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_55_memoryconformance_operate_remember_dedups_core_by_tags(): +async def test_vector_86_memoryconformance_operate_remember_dedups_core_by_tags(): vector_json = r""" { "name": "remember_dedups_core_by_tags", @@ -2135,7 +3077,7 @@ async def test_vector_55_memoryconformance_operate_remember_dedups_core_by_tags( await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_56_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): +async def test_vector_87_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags(): vector_json = r""" { "name": "remember_dedups_core_missing_and_empty_tags", @@ -2189,7 +3131,7 @@ async def test_vector_56_memoryconformance_operate_remember_dedups_core_missing_ await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_57_memoryconformance_operate_remember_evicts_archival_first(): +async def test_vector_88_memoryconformance_operate_remember_evicts_archival_first(): vector_json = r""" { "name": "remember_evicts_archival_first", @@ -2248,7 +3190,7 @@ async def test_vector_57_memoryconformance_operate_remember_evicts_archival_firs await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_58_memoryconformance_operate_snapshot_port_round_trip(): +async def test_vector_89_memoryconformance_operate_snapshot_port_round_trip(): vector_json = r""" { "name": "snapshot_port_round_trip", @@ -2286,7 +3228,7 @@ async def test_vector_58_memoryconformance_operate_snapshot_port_round_trip(): await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_59_memoryconformance_operate_update_index_out_of_range_error(): +async def test_vector_90_memoryconformance_operate_update_index_out_of_range_error(): vector_json = r""" { "name": "update_index_out_of_range_error", @@ -2318,7 +3260,7 @@ async def test_vector_59_memoryconformance_operate_update_index_out_of_range_err await run_vector("MemoryConformance", "operate", vector, False, _SEAM) -async def test_vector_60_turnconformance_replay_max_iterations(): +async def test_vector_91_turnconformance_replay_max_iterations(): vector_json = r""" { "name": "max_iterations", @@ -2353,7 +3295,7 @@ async def test_vector_60_turnconformance_replay_max_iterations(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_61_turnconformance_replay_no_tool(): +async def test_vector_92_turnconformance_replay_no_tool(): vector_json = r""" { "name": "no_tool", @@ -2384,7 +3326,7 @@ async def test_vector_61_turnconformance_replay_no_tool(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_62_turnconformance_replay_permission_denied(): +async def test_vector_93_turnconformance_replay_permission_denied(): vector_json = r""" { "name": "permission_denied", @@ -2418,7 +3360,7 @@ async def test_vector_62_turnconformance_replay_permission_denied(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_63_turnconformance_replay_tool_failure(): +async def test_vector_94_turnconformance_replay_tool_failure(): vector_json = r""" { "name": "tool_failure", @@ -2454,7 +3396,7 @@ async def test_vector_63_turnconformance_replay_tool_failure(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_64_turnconformance_replay_tool_success(): +async def test_vector_95_turnconformance_replay_tool_success(): vector_json = r""" { "name": "tool_success", @@ -2490,7 +3432,7 @@ async def test_vector_64_turnconformance_replay_tool_success(): await run_vector("TurnConformance", "replay", vector, False, _SEAM) -async def test_vector_65_turnconformance_run_assistant_tool_calls_metadata(): +async def test_vector_96_turnconformance_run_assistant_tool_calls_metadata(): vector_json = r""" { "name": "assistant_tool_calls_metadata", @@ -2642,7 +3584,7 @@ async def test_vector_65_turnconformance_run_assistant_tool_calls_metadata(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_66_turnconformance_run_async_tool_function(): +async def test_vector_97_turnconformance_run_async_tool_function(): vector_json = r""" { "name": "async_tool_function", @@ -2752,7 +3694,7 @@ async def test_vector_66_turnconformance_run_async_tool_function(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_67_turnconformance_run_bindings_injected(): +async def test_vector_98_turnconformance_run_bindings_injected(): vector_json = r""" { "name": "bindings_injected", @@ -2882,7 +3824,7 @@ async def test_vector_67_turnconformance_run_bindings_injected(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_68_turnconformance_run_cancellation_before_llm(): +async def test_vector_99_turnconformance_run_cancellation_before_llm(): vector_json = r""" { "name": "cancellation_before_llm", @@ -2943,7 +3885,7 @@ async def test_vector_68_turnconformance_run_cancellation_before_llm(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_69_turnconformance_run_cancellation_between_iterations(): +async def test_vector_100_turnconformance_run_cancellation_between_iterations(): vector_json = r""" { "name": "cancellation_between_iterations", @@ -3073,7 +4015,7 @@ async def test_vector_69_turnconformance_run_cancellation_between_iterations(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_70_turnconformance_run_cancellation_between_tools(): +async def test_vector_101_turnconformance_run_cancellation_between_tools(): vector_json = r""" { "name": "cancellation_between_tools", @@ -3212,7 +4154,7 @@ async def test_vector_70_turnconformance_run_cancellation_between_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_71_turnconformance_run_context_no_trim_when_fits(): +async def test_vector_102_turnconformance_run_context_no_trim_when_fits(): vector_json = r""" { "name": "context_no_trim_when_fits", @@ -3283,7 +4225,7 @@ async def test_vector_71_turnconformance_run_context_no_trim_when_fits(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_72_turnconformance_run_context_preserves_system_messages(): +async def test_vector_103_turnconformance_run_context_preserves_system_messages(): vector_json = r""" { "name": "context_preserves_system_messages", @@ -3390,7 +4332,7 @@ async def test_vector_72_turnconformance_run_context_preserves_system_messages() await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_73_turnconformance_run_context_trim_basic(): +async def test_vector_104_turnconformance_run_context_trim_basic(): vector_json = r""" { "name": "context_trim_basic", @@ -3548,7 +4490,7 @@ async def test_vector_73_turnconformance_run_context_trim_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_74_turnconformance_run_empty_tool_result(): +async def test_vector_105_turnconformance_run_empty_tool_result(): vector_json = r""" { "name": "empty_tool_result", @@ -3663,7 +4605,7 @@ async def test_vector_74_turnconformance_run_empty_tool_result(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_75_turnconformance_run_events_basic_tool_loop(): +async def test_vector_106_turnconformance_run_events_basic_tool_loop(): vector_json = r""" { "name": "events_basic_tool_loop", @@ -3809,7 +4751,7 @@ async def test_vector_75_turnconformance_run_events_basic_tool_loop(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_76_turnconformance_run_events_error_logged(): +async def test_vector_107_turnconformance_run_events_error_logged(): vector_json = r""" { "name": "events_error_logged", @@ -3935,7 +4877,7 @@ async def test_vector_76_turnconformance_run_events_error_logged(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_77_turnconformance_run_events_no_tools(): +async def test_vector_108_turnconformance_run_events_no_tools(): vector_json = r""" { "name": "events_no_tools", @@ -4013,7 +4955,7 @@ async def test_vector_77_turnconformance_run_events_no_tools(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_78_turnconformance_run_guardrail_all_pass(): +async def test_vector_109_turnconformance_run_guardrail_all_pass(): vector_json = r""" { "name": "guardrail_all_pass", @@ -4136,7 +5078,7 @@ async def test_vector_78_turnconformance_run_guardrail_all_pass(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_79_turnconformance_run_guardrail_input_deny(): +async def test_vector_110_turnconformance_run_guardrail_input_deny(): vector_json = r""" { "name": "guardrail_input_deny", @@ -4192,7 +5134,7 @@ async def test_vector_79_turnconformance_run_guardrail_input_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_80_turnconformance_run_guardrail_output_deny(): +async def test_vector_111_turnconformance_run_guardrail_output_deny(): vector_json = r""" { "name": "guardrail_output_deny", @@ -4270,7 +5212,7 @@ async def test_vector_80_turnconformance_run_guardrail_output_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_81_turnconformance_run_guardrail_tool_deny(): +async def test_vector_112_turnconformance_run_guardrail_tool_deny(): vector_json = r""" { "name": "guardrail_tool_deny", @@ -4424,7 +5366,7 @@ async def test_vector_81_turnconformance_run_guardrail_tool_deny(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_82_turnconformance_run_max_iterations_exceeded(): +async def test_vector_113_turnconformance_run_max_iterations_exceeded(): vector_json = r""" { "name": "max_iterations_exceeded", @@ -4934,7 +5876,7 @@ async def test_vector_82_turnconformance_run_max_iterations_exceeded(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_83_turnconformance_run_multi_turn_tool_calls(): +async def test_vector_114_turnconformance_run_multi_turn_tool_calls(): vector_json = r""" { "name": "multi_turn_tool_calls", @@ -5111,7 +6053,7 @@ async def test_vector_83_turnconformance_run_multi_turn_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_84_turnconformance_run_multiple_tool_calls_single_turn(): +async def test_vector_115_turnconformance_run_multiple_tool_calls_single_turn(): vector_json = r""" { "name": "multiple_tool_calls_single_turn", @@ -5292,7 +6234,7 @@ async def test_vector_84_turnconformance_run_multiple_tool_calls_single_turn(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_85_turnconformance_run_no_tool_calls(): +async def test_vector_116_turnconformance_run_no_tool_calls(): vector_json = r""" { "name": "no_tool_calls", @@ -5360,7 +6302,7 @@ async def test_vector_85_turnconformance_run_no_tool_calls(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_86_turnconformance_run_parallel_tools_basic(): +async def test_vector_117_turnconformance_run_parallel_tools_basic(): vector_json = r""" { "name": "parallel_tools_basic", @@ -5533,7 +6475,7 @@ async def test_vector_86_turnconformance_run_parallel_tools_basic(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_87_turnconformance_run_parallel_tools_with_guardrail_deny(): +async def test_vector_118_turnconformance_run_parallel_tools_with_guardrail_deny(): vector_json = r""" { "name": "parallel_tools_with_guardrail_deny", @@ -5724,7 +6666,7 @@ async def test_vector_87_turnconformance_run_parallel_tools_with_guardrail_deny( await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_88_turnconformance_run_parallel_tools_with_tool_error(): +async def test_vector_119_turnconformance_run_parallel_tools_with_tool_error(): vector_json = r""" { "name": "parallel_tools_with_tool_error", @@ -5897,7 +6839,7 @@ async def test_vector_88_turnconformance_run_parallel_tools_with_tool_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_89_turnconformance_run_single_tool_call(): +async def test_vector_120_turnconformance_run_single_tool_call(): vector_json = r""" { "name": "single_tool_call", @@ -6044,7 +6986,7 @@ async def test_vector_89_turnconformance_run_single_tool_call(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_90_turnconformance_run_steering_inject_message(): +async def test_vector_121_turnconformance_run_steering_inject_message(): vector_json = r""" { "name": "steering_inject_message", @@ -6252,7 +7194,7 @@ async def test_vector_90_turnconformance_run_steering_inject_message(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_91_turnconformance_run_steering_multiple_messages(): +async def test_vector_122_turnconformance_run_steering_multiple_messages(): vector_json = r""" { "name": "steering_multiple_messages", @@ -6422,7 +7364,7 @@ async def test_vector_91_turnconformance_run_steering_multiple_messages(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_92_turnconformance_run_tool_not_registered_error(): +async def test_vector_123_turnconformance_run_tool_not_registered_error(): vector_json = r""" { "name": "tool_not_registered_error", @@ -6506,7 +7448,7 @@ async def test_vector_92_turnconformance_run_tool_not_registered_error(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_93_turnconformance_run_tool_result_message_format(): +async def test_vector_124_turnconformance_run_tool_result_message_format(): vector_json = r""" { "name": "tool_result_message_format", @@ -6627,7 +7569,7 @@ async def test_vector_93_turnconformance_run_tool_result_message_format(): await run_vector("TurnConformance", "run", vector, False, _SEAM) -async def test_vector_94_turnconformance_runturn_cancel_before_context(): +async def test_vector_125_turnconformance_runturn_cancel_before_context(): vector_json = r""" { "name": "cancel_before_context", @@ -6659,7 +7601,7 @@ async def test_vector_94_turnconformance_runturn_cancel_before_context(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_95_turnconformance_runturn_delegated_provider_state(): +async def test_vector_126_turnconformance_runturn_delegated_provider_state(): vector_json = r""" { "name": "delegated_provider_state", @@ -6726,7 +7668,7 @@ async def test_vector_95_turnconformance_runturn_delegated_provider_state(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_96_turnconformance_runturn_final_output(): +async def test_vector_127_turnconformance_runturn_final_output(): vector_json = r""" { "name": "final_output", @@ -6771,7 +7713,7 @@ async def test_vector_96_turnconformance_runturn_final_output(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_97_turnconformance_runturn_memory_context_is_model_visible(): +async def test_vector_128_turnconformance_runturn_memory_context_is_model_visible(): vector_json = r""" { "name": "memory_context_is_model_visible", @@ -6833,7 +7775,7 @@ async def test_vector_97_turnconformance_runturn_memory_context_is_model_visible await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_98_turnconformance_runturn_ordered_tool_round(): +async def test_vector_129_turnconformance_runturn_ordered_tool_round(): vector_json = r""" { "name": "ordered_tool_round", @@ -6924,7 +7866,7 @@ async def test_vector_98_turnconformance_runturn_ordered_tool_round(): await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_99_turnconformance_runturn_permission_denial_is_model_visible(): +async def test_vector_130_turnconformance_runturn_permission_denial_is_model_visible(): vector_json = r""" { "name": "permission_denial_is_model_visible", @@ -6975,7 +7917,7 @@ async def test_vector_99_turnconformance_runturn_permission_denial_is_model_visi await run_vector("TurnConformance", "runTurn", vector, False, _SEAM) -async def test_vector_100_wireconformance_torequest_anthropic_image_format(): +async def test_vector_131_wireconformance_torequest_anthropic_image_format(): vector_json = r""" { "name": "anthropic_image_format", @@ -7045,7 +7987,7 @@ async def test_vector_100_wireconformance_torequest_anthropic_image_format(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_101_wireconformance_torequest_anthropic_max_tokens_required(): +async def test_vector_132_wireconformance_torequest_anthropic_max_tokens_required(): vector_json = r""" { "name": "anthropic_max_tokens_required", @@ -7100,7 +8042,7 @@ async def test_vector_101_wireconformance_torequest_anthropic_max_tokens_require await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_102_wireconformance_torequest_anthropic_options(): +async def test_vector_133_wireconformance_torequest_anthropic_options(): vector_json = r""" { "name": "anthropic_options", @@ -7169,7 +8111,7 @@ async def test_vector_102_wireconformance_torequest_anthropic_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_103_wireconformance_torequest_anthropic_system_separate(): +async def test_vector_134_wireconformance_torequest_anthropic_system_separate(): vector_json = r""" { "name": "anthropic_system_separate", @@ -7236,7 +8178,7 @@ async def test_vector_103_wireconformance_torequest_anthropic_system_separate(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_104_wireconformance_torequest_anthropic_tool_wire(): +async def test_vector_135_wireconformance_torequest_anthropic_tool_wire(): vector_json = r""" { "name": "anthropic_tool_wire", @@ -7321,7 +8263,7 @@ async def test_vector_104_wireconformance_torequest_anthropic_tool_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_105_wireconformance_torequest_chat_audio_mp3(): +async def test_vector_136_wireconformance_torequest_chat_audio_mp3(): vector_json = r""" { "name": "chat_audio_mp3", @@ -7379,7 +8321,7 @@ async def test_vector_105_wireconformance_torequest_chat_audio_mp3(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_106_wireconformance_torequest_chat_audio_part(): +async def test_vector_137_wireconformance_torequest_chat_audio_part(): vector_json = r""" { "name": "chat_audio_part", @@ -7437,7 +8379,7 @@ async def test_vector_106_wireconformance_torequest_chat_audio_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_107_wireconformance_torequest_chat_image_base64(): +async def test_vector_138_wireconformance_torequest_chat_image_base64(): vector_json = r""" { "name": "chat_image_base64", @@ -7501,7 +8443,7 @@ async def test_vector_107_wireconformance_torequest_chat_image_base64(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_108_wireconformance_torequest_chat_image_part(): +async def test_vector_139_wireconformance_torequest_chat_image_part(): vector_json = r""" { "name": "chat_image_part", @@ -7565,7 +8507,7 @@ async def test_vector_108_wireconformance_torequest_chat_image_part(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_109_wireconformance_torequest_chat_multipart_content(): +async def test_vector_140_wireconformance_torequest_chat_multipart_content(): vector_json = r""" { "name": "chat_multipart_content", @@ -7629,7 +8571,7 @@ async def test_vector_109_wireconformance_torequest_chat_multipart_content(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_110_wireconformance_torequest_chat_simple(): +async def test_vector_141_wireconformance_torequest_chat_simple(): vector_json = r""" { "name": "chat_simple", @@ -7691,7 +8633,7 @@ async def test_vector_110_wireconformance_torequest_chat_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_111_wireconformance_torequest_chat_single_text_optimized(): +async def test_vector_142_wireconformance_torequest_chat_single_text_optimized(): vector_json = r""" { "name": "chat_single_text_optimized", @@ -7740,7 +8682,7 @@ async def test_vector_111_wireconformance_torequest_chat_single_text_optimized() await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_112_wireconformance_torequest_chat_with_options(): +async def test_vector_143_wireconformance_torequest_chat_with_options(): vector_json = r""" { "name": "chat_with_options", @@ -7808,7 +8750,7 @@ async def test_vector_112_wireconformance_torequest_chat_with_options(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_113_wireconformance_torequest_embedding_wire(): +async def test_vector_144_wireconformance_torequest_embedding_wire(): vector_json = r""" { "name": "embedding_wire", @@ -7852,7 +8794,7 @@ async def test_vector_113_wireconformance_torequest_embedding_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_114_wireconformance_torequest_image_wire(): +async def test_vector_145_wireconformance_torequest_image_wire(): vector_json = r""" { "name": "image_wire", @@ -7896,7 +8838,7 @@ async def test_vector_114_wireconformance_torequest_image_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_115_wireconformance_torequest_kind_to_json_type_mapping(): +async def test_vector_146_wireconformance_torequest_kind_to_json_type_mapping(): vector_json = r""" { "name": "kind_to_json_type_mapping", @@ -8015,7 +8957,7 @@ async def test_vector_115_wireconformance_torequest_kind_to_json_type_mapping(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_116_wireconformance_torequest_options_additional_properties(): +async def test_vector_147_wireconformance_torequest_options_additional_properties(): vector_json = r""" { "name": "options_additional_properties", @@ -8069,7 +9011,7 @@ async def test_vector_116_wireconformance_torequest_options_additional_propertie await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_117_wireconformance_torequest_options_max_completion_tokens(): +async def test_vector_148_wireconformance_torequest_options_max_completion_tokens(): vector_json = r""" { "name": "options_max_completion_tokens", @@ -8121,7 +9063,7 @@ async def test_vector_117_wireconformance_torequest_options_max_completion_token await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_118_wireconformance_torequest_options_stop_sequences(): +async def test_vector_149_wireconformance_torequest_options_stop_sequences(): vector_json = r""" { "name": "options_stop_sequences", @@ -8179,7 +9121,7 @@ async def test_vector_118_wireconformance_torequest_options_stop_sequences(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_119_wireconformance_torequest_responses_simple(): +async def test_vector_150_wireconformance_torequest_responses_simple(): vector_json = r""" { "name": "responses_simple", @@ -8238,7 +9180,7 @@ async def test_vector_119_wireconformance_torequest_responses_simple(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_120_wireconformance_torequest_responses_structured_output(): +async def test_vector_151_wireconformance_torequest_responses_structured_output(): vector_json = r""" { "name": "responses_structured_output", @@ -8325,7 +9267,7 @@ async def test_vector_120_wireconformance_torequest_responses_structured_output( await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_121_wireconformance_torequest_responses_with_tools(): +async def test_vector_152_wireconformance_torequest_responses_with_tools(): vector_json = r""" { "name": "responses_with_tools", @@ -8405,7 +9347,7 @@ async def test_vector_121_wireconformance_torequest_responses_with_tools(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_122_wireconformance_torequest_structured_output(): +async def test_vector_153_wireconformance_torequest_structured_output(): vector_json = r""" { "name": "structured_output", @@ -8488,7 +9430,7 @@ async def test_vector_122_wireconformance_torequest_structured_output(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_123_wireconformance_torequest_structured_output_nested_optional(): +async def test_vector_154_wireconformance_torequest_structured_output_nested_optional(): vector_json = r""" { "name": "structured_output_nested_optional", @@ -8590,7 +9532,7 @@ async def test_vector_123_wireconformance_torequest_structured_output_nested_opt await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_124_wireconformance_torequest_tools_bindings_stripped(): +async def test_vector_155_wireconformance_torequest_tools_bindings_stripped(): vector_json = r""" { "name": "tools_bindings_stripped", @@ -8682,7 +9624,7 @@ async def test_vector_124_wireconformance_torequest_tools_bindings_stripped(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_125_wireconformance_torequest_tools_function_wire(): +async def test_vector_156_wireconformance_torequest_tools_function_wire(): vector_json = r""" { "name": "tools_function_wire", @@ -8764,7 +9706,7 @@ async def test_vector_125_wireconformance_torequest_tools_function_wire(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_126_wireconformance_torequest_tools_null_when_empty(): +async def test_vector_157_wireconformance_torequest_tools_null_when_empty(): vector_json = r""" { "name": "tools_null_when_empty", @@ -8813,7 +9755,7 @@ async def test_vector_126_wireconformance_torequest_tools_null_when_empty(): await run_vector("WireConformance", "toRequest", vector, False, _SEAM) -async def test_vector_127_wireconformance_torequest_tools_strict_mode(): +async def test_vector_158_wireconformance_torequest_tools_strict_mode(): vector_json = r""" { "name": "tools_strict_mode", diff --git a/runtime/python/prompty/tests/model/vector_adapters.py b/runtime/python/prompty/tests/model/vector_adapters.py index 9de184a54..b5076e013 100644 --- a/runtime/python/prompty/tests/model/vector_adapters.py +++ b/runtime/python/prompty/tests/model/vector_adapters.py @@ -271,6 +271,11 @@ def _load_invoke(input: dict, context: dict) -> Any: if "NONEXISTENT" in json.dumps(input): old_env.setdefault("NONEXISTENT", os.environ.get("NONEXISTENT")) os.environ.pop("NONEXISTENT", None) + # Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + if "PROMPTY_TRUSTED_ENDPOINTS" not in env_vars: + old_env.setdefault("PROMPTY_TRUSTED_ENDPOINTS", os.environ.get("PROMPTY_TRUSTED_ENDPOINTS")) + os.environ.pop("PROMPTY_TRUSTED_ENDPOINTS", None) + trusted = input.get("trustedEndpoints") try: # --- input-validation vectors --- @@ -282,18 +287,18 @@ def _load_invoke(input: dict, context: dict) -> Any: with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) if "fixture" in input: - agent = load(SPEC_FIXTURES / input["fixture"]) + agent = load(SPEC_FIXTURES / input["fixture"], trusted_endpoints=trusted) elif "frontmatter_raw" in input: p = tmp_path / "vector.prompty" p.write_text(input["frontmatter_raw"], encoding="utf-8") - agent = load(p) + agent = load(p, trusted_endpoints=trusted) else: frontmatter = dict(input["frontmatter"]) sub = tmp_path / input["agent_subdir"] if input.get("agent_subdir") else tmp_path sub.mkdir(parents=True, exist_ok=True) p = sub / "vector.prompty" _write_prompty(p, frontmatter, input.get("files")) - agent = load(p) + agent = load(p, trusted_endpoints=trusted) return _agent_to_canonical(agent.save(default_save_context(use_shorthand=False))) except Exception as exc: # noqa: BLE001 # Error vectors: attach the canonical {kind, [field]} so the harness can diff --git a/runtime/python/prompty/tests/test_endpoint_trust.py b/runtime/python/prompty/tests/test_endpoint_trust.py new file mode 100644 index 000000000..c4517c86b --- /dev/null +++ b/runtime/python/prompty/tests/test_endpoint_trust.py @@ -0,0 +1,150 @@ +"""Tests for endpoint trust (spec §2.5.1).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from prompty import load +from prompty.core.endpoint_trust import is_trusted_endpoint +from prompty.core.errors import PromptyLoadError + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PROMPTY_TRUSTED_ENDPOINTS", raising=False) + + +def _write(tmp_path: Path, connection: str) -> Path: + p = tmp_path / "t.prompty" + p.write_text(f"---\nname: t\nmodel:\n id: gpt-4\n connection:\n{connection}\n---\nHello\n", encoding="utf-8") + return p + + +class TestIsTrustedEndpoint: + @pytest.mark.parametrize( + "url", + [ + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + ], + ) + def test_builtin_trusted(self, url: str): + assert is_trusted_endpoint(url) + + @pytest.mark.parametrize( + "url", + [ + "http://api.openai.com", # scheme mismatch + "https://api.openai.com:8443", # port mismatch + "https://openai.azure.com", # wildcard never matches apex + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", # userinfo ignored; host is evil.com + "https://evil.com\\@api.openai.com", # backslash: clients connect to evil.com + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "", + ], + ) + def test_untrusted(self, url: str): + assert not is_trusted_endpoint(url) + + def test_host_option_and_wildcard(self): + assert is_trusted_endpoint("http://localhost:11434/v1", ["http://LOCALHOST:11434"]) + assert is_trusted_endpoint("https://x.contoso.com", ["https://*.contoso.com"]) + assert not is_trusted_endpoint("https://contoso.com", ["https://*.contoso.com"]) + + def test_env_list_and_star(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("PROMPTY_TRUSTED_ENDPOINTS", " https://a.example.com , http://localhost:8080") + assert is_trusted_endpoint("http://localhost:8080/x") + assert not is_trusted_endpoint("http://localhost:8081") + monkeypatch.setenv("PROMPTY_TRUSTED_ENDPOINTS", "*") + assert is_trusted_endpoint("http://anything.example") + + +class TestLoaderEnforcement: + def test_ambient_literal_endpoint_rejected(self, tmp_path: Path): + p = _write(tmp_path, " kind: foundry\n endpoint: http://127.0.0.1:9999") + with pytest.raises(PromptyLoadError) as exc: + load(p) + assert exc.value.kind == "untrusted_endpoint" + assert "127.0.0.1" in str(exc.value) + + def test_env_key_literal_endpoint_rejected(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("SOME_KEY", "secret") + p = _write(tmp_path, " kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}") + with pytest.raises(PromptyLoadError, match="not trusted"): + load(p) + + def test_oauth_token_url_checked(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("SECRET", "s") + conn = ( + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token" + ) + with pytest.raises(PromptyLoadError, match="tokenUrl"): + load(_write(tmp_path, conn)) + + def test_mismatched_credential_field_rejected(self, tmp_path: Path): + p = _write(tmp_path, " kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk") + with pytest.raises(PromptyLoadError, match="not trusted"): + load(p) + p = _write(tmp_path, " kind: key\n endpoint: https://evil.example.com\n clientSecret: junk") + with pytest.raises(PromptyLoadError, match="not trusted"): + load(p) + + def test_literal_key_allowed(self, tmp_path: Path): + agent = load(_write(tmp_path, " kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine")) + assert agent.model.connection.endpoint == "http://127.0.0.1:9999" + + def test_env_endpoint_allowed(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("EP", "http://127.0.0.1:9999") + agent = load(_write(tmp_path, " kind: foundry\n endpoint: ${env:EP}")) + assert agent.model.connection.endpoint == "http://127.0.0.1:9999" + + def test_env_default_used_rejected(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("UNSET_EP", raising=False) + with pytest.raises(PromptyLoadError, match="not trusted"): + load(_write(tmp_path, " kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}")) + + def test_trusted_option_allows(self, tmp_path: Path): + p = _write(tmp_path, " kind: foundry\n endpoint: http://127.0.0.1:9999/api") + agent = load(p, trusted_endpoints=["http://127.0.0.1:9999"]) + assert agent.model.connection.endpoint == "http://127.0.0.1:9999/api" + + @pytest.mark.asyncio + async def test_async_honors_option(self, tmp_path: Path): + from prompty import load_async + + p = _write(tmp_path, " kind: foundry\n endpoint: http://127.0.0.1:9999") + with pytest.raises(PromptyLoadError): + await load_async(p) + agent = await load_async(p, trusted_endpoints=["http://127.0.0.1:9999"]) + assert agent.model.connection.endpoint == "http://127.0.0.1:9999" + + def test_file_model_reference_checked(self, tmp_path: Path): + (tmp_path / "model.json").write_text( + json.dumps({"id": "gpt-4", "connection": {"kind": "foundry", "endpoint": "https://evil.example.com"}}), + encoding="utf-8", + ) + p = tmp_path / "t.prompty" + p.write_text("---\nname: t\nmodel: ${file:model.json}\n---\nHi\n", encoding="utf-8") + with pytest.raises(PromptyLoadError, match="not trusted"): + load(p) + + def test_frontmatter_cannot_self_trust(self, tmp_path: Path): + p = tmp_path / "t.prompty" + p.write_text( + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n", + encoding="utf-8", + ) + with pytest.raises(PromptyLoadError): + load(p) diff --git a/runtime/rust/prompty/README.md b/runtime/rust/prompty/README.md index 602afb338..7e3266792 100644 --- a/runtime/rust/prompty/README.md +++ b/runtime/rust/prompty/README.md @@ -40,6 +40,33 @@ structured output. `TurnOptions::validator` runs on that final payload after any guardrail rewrite and before the durable success commit; rejection records a failed turn. `max_llm_retries` controls model attempts for both simple and tool-calling turns. +### Load options and endpoint trust + +`load_with_options()` / `load_async_with_options()` accept `LoadOptions`: + +- `allowed_file_roots` — extra directories `${file:...}` references may read from. +- `trusted_endpoints` — extra origins allowed to receive ambient credentials. + +A `.prompty` file is untrusted. When its connection has no literal file-supplied +credential (a literal `apiKey` on `kind: key`), executors fall +back to ambient credentials (Entra ID tokens, `*_API_KEY` env vars). Loading fails with +`LoadError::UntrustedEndpoint` if such a connection's `endpoint` / `tokenUrl` is chosen +by the file (a literal, `${file:...}`, or an `${env:VAR:default}` default) and its origin +is not trusted. Trusted origins are the built-in provider origins (`https://api.openai.com`, +`https://api.anthropic.com`, `https://*.openai.azure.com`, `https://*.services.ai.azure.com`, +`https://*.cognitiveservices.azure.com`, `https://login.microsoftonline.com`), plus +`trusted_endpoints`, plus the comma-separated `PROMPTY_TRUSTED_ENDPOINTS` env var (`*` +trusts everything). Endpoints supplied via `${env:VAR}` with `VAR` set are host-controlled +and always allowed. + +```rust +let options = prompty::LoadOptions { + trusted_endpoints: vec!["http://localhost:11434".into()], + ..Default::default() +}; +let agent = prompty::load_with_options("local.prompty", options)?; +``` + ## Live turns, durability, and resume `turn()` is the convenient live API. For hosts that own a session journal or need to diff --git a/runtime/rust/prompty/src/loader.rs b/runtime/rust/prompty/src/loader.rs index 9f730b56a..ba3359deb 100644 --- a/runtime/rust/prompty/src/loader.rs +++ b/runtime/rust/prompty/src/loader.rs @@ -9,6 +9,7 @@ use std::path::{Path, PathBuf}; use crate::model::Agent; use crate::model::context::LoadContext; +pub mod endpoint_trust; mod error; mod frontmatter; mod resolve; @@ -21,6 +22,11 @@ pub struct LoadOptions { /// Additional directories that `${file:...}` references may read from. /// The prompt file's directory is always allowed. pub allowed_file_roots: Vec, + /// Additional origins (e.g. `https://my-proxy.contoso.com`, + /// `https://*.contoso.com`, or `*`) trusted to receive ambient credentials + /// (spec §2.5.1). Built-in provider origins and the comma-separated + /// `PROMPTY_TRUSTED_ENDPOINTS` env var are always included. + pub trusted_endpoints: Vec, } /// Load a `.prompty` file and return a typed `Prompty`. @@ -176,12 +182,24 @@ fn build_agent(raw: &str, file_path: &Path, options: &LoadOptions) -> Result Result Option { + match scheme { + "http" => Some(80), + "https" => Some(443), + _ => None, + } +} + +/// Return `(scheme, host, port)` for an absolute http(s) URL, else `None`. +/// +/// Mirrors Python's `urllib.parse.urlsplit` semantics used by the reference +/// implementation; additionally treats a backslash in the authority as +/// unparseable, since WHATWG-style HTTP clients read it as a path separator. +fn origin(url: &str) -> Option<(String, String, u16)> { + let cleaned: String = url + .trim() + .trim_start_matches(|c: char| c <= ' ') + .chars() + .filter(|c| !matches!(c, '\t' | '\r' | '\n')) + .collect(); + + let colon = cleaned.find(':')?; + let scheme = &cleaned[..colon]; + let mut chars = scheme.chars(); + let first = chars.next()?; + if !first.is_ascii_alphabetic() + || !scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + { + return None; + } + let scheme = scheme.to_ascii_lowercase(); + let default = default_port(&scheme)?; + + let rest = cleaned[colon + 1..].strip_prefix("//")?; + let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let netloc = &rest[..end]; + if netloc.contains('\\') || netloc.contains('[') != netloc.contains(']') { + return None; + } + + let hostinfo = netloc.rsplit_once('@').map_or(netloc, |(_, h)| h); + let (host, port) = match hostinfo.split_once('[') { + Some((_, bracketed)) => { + let (host, after) = bracketed.split_once(']').unwrap_or((bracketed, "")); + if host.parse::().is_err() { + return None; + } + (host, after.split_once(':').map_or("", |(_, p)| p)) + } + None => hostinfo.split_once(':').unwrap_or((hostinfo, "")), + }; + let host = host.to_lowercase(); + if host.is_empty() { + return None; + } + let port = if port.is_empty() { + default + } else { + if !port.chars().all(|c| c.is_ascii_digit()) { + return None; + } + port.parse::().ok()? + }; + Some((scheme, host, port)) +} + +fn trusted_entries(trusted_endpoints: &[String]) -> Vec { + let env = std::env::var(TRUSTED_ENDPOINTS_ENV).unwrap_or_default(); + BUILTIN_TRUSTED_ENDPOINTS + .iter() + .map(|s| s.to_string()) + .chain(trusted_endpoints.iter().cloned()) + .chain(env.split(',').map(str::to_string)) + .map(|e| e.trim().to_string()) + .filter(|e| !e.is_empty()) + .collect() +} + +/// Return `true` if `url`'s origin matches a built-in, host, or env trusted origin. +pub fn is_trusted_endpoint(url: &str, trusted_endpoints: &[String]) -> bool { + let entries = trusted_entries(trusted_endpoints); + if entries.iter().any(|e| e == "*") { + return true; + } + let Some((scheme, host, port)) = origin(url) else { + return false; + }; + entries.iter().any(|entry| { + let Some((a_scheme, pattern, a_port)) = origin(entry) else { + return false; + }; + if a_scheme != scheme || a_port != port { + return false; + } + match pattern.strip_prefix('*') { + Some(suffix) if suffix.starts_with('.') => host.ends_with(suffix), + _ => pattern == host, + } + }) +} + +fn is_reference(value: &str) -> bool { + value.starts_with("${") && value.ends_with('}') +} + +fn file_supplied_credential(connection: &serde_json::Map) -> bool { + // Only the credential field the declared kind actually consumes counts; any other kind is ambient. + let key = match connection.get("kind").and_then(Value::as_str) { + Some("key") => "apiKey", + + _ => return false, + }; + // Blank keys count as absent: executors that trim fall back to ambient credentials. + matches!(connection.get(key), Some(Value::String(s)) if !s.trim().is_empty() && !is_reference(s)) +} + +/// Return the effective destination if the file chose it, else `None`. +fn file_controlled_destination(value: Option<&Value>) -> Option { + let value = match value? { + Value::Null => return None, + Value::String(s) => s, + other => return Some(other.to_string()), + }; + if is_reference(value) { + let inner = &value[2..value.len() - 1]; + let (protocol, rest) = inner.split_once(':').unwrap_or((inner, "")); + if protocol.eq_ignore_ascii_case("env") { + let (var_name, default) = rest.split_once(':').unwrap_or((rest, "")); + if std::env::var_os(var_name).is_some() || default.is_empty() { + return None; + } + return Some(default.to_string()); + } + } + Some(value.clone()) +} + +/// Return [`LoadError::UntrustedEndpoint`] if ambient credentials could reach a +/// file-chosen, untrusted destination. +/// +/// `connection` is the RAW (unresolved) `model.connection` value. +pub fn check_connection_trust( + connection: Option<&Value>, + trusted_endpoints: &[String], +) -> Result<(), LoadError> { + let Some(Value::Object(connection)) = connection else { + return Ok(()); + }; + if connection.get("kind").and_then(Value::as_str) == Some("reference") + || file_supplied_credential(connection) + { + return Ok(()); + } + for key in DESTINATION_KEYS { + if let Some(destination) = file_controlled_destination(connection.get(key)) { + if !is_trusted_endpoint(&destination, trusted_endpoints) { + return Err(LoadError::UntrustedEndpoint(format!( + "Connection {key} '{destination}' is not trusted to receive ambient credentials. \ + Set the {key} via ${{env:VAR}}, add its origin to trusted_endpoints or \ + {TRUSTED_ENDPOINTS_ENV}, or use 'kind: reference'." + ))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::loader::{LoadOptions, load_with_options}; + use serial_test::serial; + use std::path::{Path, PathBuf}; + + struct EnvGuard(Vec<(&'static str, Option)>); + + impl EnvGuard { + fn new(vars: &[(&'static str, Option<&str>)]) -> Self { + let mut saved = vec![( + TRUSTED_ENDPOINTS_ENV, + std::env::var_os(TRUSTED_ENDPOINTS_ENV), + )]; + unsafe { std::env::remove_var(TRUSTED_ENDPOINTS_ENV) }; + for (k, v) in vars { + saved.push((k, std::env::var_os(k))); + match v { + Some(v) => unsafe { std::env::set_var(k, v) }, + None => unsafe { std::env::remove_var(k) }, + } + } + EnvGuard(saved) + } + } + + impl Drop for EnvGuard { + fn drop(&mut self) { + for (k, v) in self.0.iter().rev() { + match v { + Some(v) => unsafe { std::env::set_var(k, v) }, + None => unsafe { std::env::remove_var(k) }, + } + } + } + } + + struct TempDir(PathBuf); + + impl TempDir { + fn new(name: &str) -> Self { + let dir = + std::env::temp_dir().join(format!("prompty_trust_{name}_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + TempDir(dir) + } + } + + impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn write(dir: &Path, connection: &str) -> PathBuf { + let p = dir.join("t.prompty"); + std::fs::write( + &p, + format!("---\nname: t\nmodel:\n id: gpt-4\n connection:\n{connection}\n---\nHello\n"), + ) + .unwrap(); + p + } + + fn load_err(p: &Path, trusted: &[&str]) -> LoadError { + let options = LoadOptions { + trusted_endpoints: trusted.iter().map(|s| s.to_string()).collect(), + ..Default::default() + }; + load_with_options(p, options).expect_err("expected load to fail") + } + + fn endpoint(p: &Path, trusted: &[&str]) -> String { + let options = LoadOptions { + trusted_endpoints: trusted.iter().map(|s| s.to_string()).collect(), + ..Default::default() + }; + let agent = load_with_options(p, options).expect("expected load to succeed"); + crate::model_access::model_connection(&agent.model)["endpoint"] + .as_str() + .unwrap_or_default() + .to_string() + } + + fn trusted(url: &str, extra: &[&str]) -> bool { + let extra: Vec = extra.iter().map(|s| s.to_string()).collect(); + is_trusted_endpoint(url, &extra) + } + + #[test] + #[serial(prompty_trusted_env)] + fn builtin_trusted() { + let _env = EnvGuard::new(&[]); + for url in [ + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + ] { + assert!(trusted(url, &[]), "{url} should be trusted"); + } + } + + #[test] + #[serial(prompty_trusted_env)] + fn untrusted() { + let _env = EnvGuard::new(&[]); + for url in [ + "http://api.openai.com", + "https://api.openai.com:8443", + "https://openai.azure.com", + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", + "https://evil.com\\@api.openai.com", + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "https://api.openai.com:99999", + "https://[::1", + "", + ] { + assert!(!trusted(url, &[]), "{url} should be untrusted"); + } + } + + #[test] + #[serial(prompty_trusted_env)] + fn host_option_and_wildcard() { + let _env = EnvGuard::new(&[]); + assert!(trusted( + "http://localhost:11434/v1", + &["http://LOCALHOST:11434"] + )); + assert!(trusted("https://x.contoso.com", &["https://*.contoso.com"])); + assert!(!trusted("https://contoso.com", &["https://*.contoso.com"])); + assert!(trusted("http://[::1]:8080/x", &["http://[::1]:8080"])); + } + + #[test] + #[serial(prompty_trusted_env)] + fn env_list_and_star() { + let _env = EnvGuard::new(&[( + TRUSTED_ENDPOINTS_ENV, + Some(" https://a.example.com , http://localhost:8080"), + )]); + assert!(trusted("http://localhost:8080/x", &[])); + assert!(!trusted("http://localhost:8081", &[])); + unsafe { std::env::set_var(TRUSTED_ENDPOINTS_ENV, "*") }; + assert!(trusted("http://anything.example", &[])); + } + + #[test] + #[serial(prompty_trusted_env)] + fn ambient_literal_endpoint_rejected() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("ambient"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: http://127.0.0.1:9999", + ); + let err = load_err(&p, &[]); + assert!(matches!(err, LoadError::UntrustedEndpoint(_))); + assert!(err.to_string().contains("127.0.0.1")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn env_key_literal_endpoint_rejected() { + let _env = EnvGuard::new(&[("PROMPTY_TRUST_SOME_KEY", Some("secret"))]); + let tmp = TempDir::new("envkey"); + let p = write( + &tmp.0, + " kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:PROMPTY_TRUST_SOME_KEY}", + ); + assert!(load_err(&p, &[]).to_string().contains("not trusted")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn oauth_token_url_checked() { + let _env = EnvGuard::new(&[("PROMPTY_TRUST_SECRET", Some("s"))]); + let tmp = TempDir::new("oauth"); + let p = write( + &tmp.0, + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n \ + clientSecret: ${env:PROMPTY_TRUST_SECRET}\n tokenUrl: https://evil.example.com/token", + ); + assert!(load_err(&p, &[]).to_string().contains("tokenUrl")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn mismatched_credential_field_rejected() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("mismatch"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk", + ); + assert!(load_err(&p, &[]).to_string().contains("not trusted")); + let p = write( + &tmp.0, + " kind: key\n endpoint: https://evil.example.com\n clientSecret: junk", + ); + assert!(load_err(&p, &[]).to_string().contains("not trusted")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn literal_key_allowed() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("literal"); + let p = write( + &tmp.0, + " kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine", + ); + assert_eq!(endpoint(&p, &[]), "http://127.0.0.1:9999"); + } + + #[test] + #[serial(prompty_trusted_env)] + fn env_endpoint_allowed() { + let _env = EnvGuard::new(&[("PROMPTY_TRUST_EP", Some("http://127.0.0.1:9999"))]); + let tmp = TempDir::new("envep"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: ${env:PROMPTY_TRUST_EP}", + ); + assert_eq!(endpoint(&p, &[]), "http://127.0.0.1:9999"); + } + + #[test] + #[serial(prompty_trusted_env)] + fn env_default_used_rejected() { + let _env = EnvGuard::new(&[("PROMPTY_TRUST_UNSET_EP", None)]); + let tmp = TempDir::new("envdefault"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: ${env:PROMPTY_TRUST_UNSET_EP:http://127.0.0.1:9999}", + ); + assert!(load_err(&p, &[]).to_string().contains("not trusted")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn trusted_option_allows() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("option"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: http://127.0.0.1:9999/api", + ); + assert_eq!( + endpoint(&p, &["http://127.0.0.1:9999"]), + "http://127.0.0.1:9999/api" + ); + } + + #[tokio::test] + #[serial(prompty_trusted_env)] + async fn async_honors_option() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("async"); + let p = write( + &tmp.0, + " kind: foundry\n endpoint: http://127.0.0.1:9999", + ); + assert!( + crate::loader::load_async(&p).await.is_err(), + "untrusted endpoint must be rejected" + ); + let options = LoadOptions { + trusted_endpoints: vec!["http://127.0.0.1:9999".to_string()], + ..Default::default() + }; + let agent = crate::loader::load_async_with_options(&p, options) + .await + .unwrap(); + assert_eq!( + crate::model_access::model_connection(&agent.model)["endpoint"] + .as_str() + .unwrap_or_default() + .to_string(), + "http://127.0.0.1:9999" + ); + } + + #[test] + #[serial(prompty_trusted_env)] + fn file_model_reference_checked() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("filemodel"); + std::fs::write( + tmp.0.join("model.json"), + r#"{"id": "gpt-4", "connection": {"kind": "foundry", "endpoint": "https://evil.example.com"}}"#, + ) + .unwrap(); + let p = tmp.0.join("t.prompty"); + std::fs::write(&p, "---\nname: t\nmodel: ${file:model.json}\n---\nHi\n").unwrap(); + assert!(load_err(&p, &[]).to_string().contains("not trusted")); + } + + #[test] + #[serial(prompty_trusted_env)] + fn file_connection_traversal_is_file_reference() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("traversal"); + let sub = tmp.0.join("prompts"); + std::fs::create_dir_all(&sub).unwrap(); + std::fs::write( + tmp.0.join("conn.json"), + r#"{"kind": "foundry", "endpoint": "https://evil.example.com"}"#, + ) + .unwrap(); + let p = sub.join("t.prompty"); + std::fs::write( + &p, + "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:../conn.json}\n---\nHi\n", + ) + .unwrap(); + assert!(matches!(load_err(&p, &[]), LoadError::FileReference { .. })); + } + + #[test] + #[serial(prompty_trusted_env)] + fn frontmatter_cannot_self_trust() { + let _env = EnvGuard::new(&[]); + let tmp = TempDir::new("selftrust"); + let p = tmp.0.join("t.prompty"); + std::fs::write( + &p, + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n\ + model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n", + ) + .unwrap(); + assert!(matches!(load_err(&p, &[]), LoadError::UntrustedEndpoint(_))); + } +} diff --git a/runtime/rust/prompty/src/loader/error.rs b/runtime/rust/prompty/src/loader/error.rs index 3e8e68df2..f8f8a2c5a 100644 --- a/runtime/rust/prompty/src/loader/error.rs +++ b/runtime/rust/prompty/src/loader/error.rs @@ -25,6 +25,9 @@ pub enum LoadError { /// Detail message detail: String, }, + /// Ambient credentials would be sent to a file-chosen, untrusted + /// destination (spec §2.5.1). + UntrustedEndpoint(String), /// A catch-all for other load errors (e.g. spawn_blocking panic). Other(String), } @@ -50,6 +53,9 @@ impl std::fmt::Display for LoadError { LoadError::FileReference { path, detail } => { write!(f, "File reference error: {}: {}", path.display(), detail) } + LoadError::UntrustedEndpoint(msg) => { + write!(f, "Untrusted endpoint: {msg}") + } LoadError::Other(msg) => { write!(f, "Load error: {msg}") } diff --git a/runtime/rust/prompty/src/loader/resolve.rs b/runtime/rust/prompty/src/loader/resolve.rs index fdd5139ed..61f91be90 100644 --- a/runtime/rust/prompty/src/loader/resolve.rs +++ b/runtime/rust/prompty/src/loader/resolve.rs @@ -57,6 +57,43 @@ pub fn resolve_single_ref( .flatten() } +/// Follow a whole-value `${file:...}` reference with the same containment +/// rules as full resolution. Non-file values are returned unchanged; a +/// referenced file that does not exist yields `Null` (full resolution reports +/// the error later). +pub fn follow_file_ref( + value: &serde_json::Value, + agent_dir: &Path, + allowed_file_roots: &[PathBuf], +) -> Result { + let Some(s) = value.as_str() else { + return Ok(value.clone()); + }; + if !s.starts_with("${") || !s.ends_with('}') { + return Ok(value.clone()); + } + let inner = &s[2..s.len() - 1]; + let Some((protocol, reference)) = inner.split_once(':') else { + return Ok(value.clone()); + }; + if !protocol.eq_ignore_ascii_case("file") { + return Ok(value.clone()); + } + let requested = Path::new(reference); + let target = if requested.is_absolute() { + requested.to_path_buf() + } else { + agent_dir.join(requested) + }; + if !target.exists() { + return Ok(serde_json::Value::Null); + } + Ok( + resolve_file(reference, agent_dir, allowed_file_roots, "model.connection")? + .unwrap_or(serde_json::Value::Null), + ) +} + /// Try to resolve a single string value. Returns `Some(resolved)` if it was /// a `${protocol:value}` reference, `None` if it should be kept as-is. fn try_resolve_string( diff --git a/runtime/rust/prompty/src/model/vector_conformance.rs b/runtime/rust/prompty/src/model/vector_conformance.rs index 1a6e2ab16..7a2ade8d0 100644 --- a/runtime/rust/prompty/src/model/vector_conformance.rs +++ b/runtime/rust/prompty/src/model/vector_conformance.rs @@ -1355,4 +1355,574 @@ pub async fn run_load_conformance_conformance Value { LE::InvalidTemplate(..) => "invalid_template", LE::EnvVarNotSet { .. } => "env_var_not_set", LE::FileReference { .. } => "file_reference", + LE::UntrustedEndpoint(..) => "untrusted_endpoint", LE::Other(..) => return serde_json::json!({ "message": err.to_string() }), }; serde_json::json!({ "kind": kind }) @@ -843,6 +844,15 @@ fn load_adapter(input: &Value, ctx: &Context) -> Result { } } } + // Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + const TRUST_ENV: &str = "PROMPTY_TRUSTED_ENDPOINTS"; + if !env_vars + .as_ref() + .is_some_and(|vars| vars.contains_key(TRUST_ENV)) + { + restore.push((TRUST_ENV.to_string(), std::env::var(TRUST_ENV).ok())); + unsafe { std::env::remove_var(TRUST_ENV) }; + } let result = load_agent_from_input(&obj, ctx); @@ -859,10 +869,23 @@ fn load_agent_from_input( obj: &serde_json::Map, ctx: &Context, ) -> Result { + let options = prompty::LoadOptions { + trusted_endpoints: obj + .get("trustedEndpoints") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|e| e.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default(), + ..Default::default() + }; + // fixture path if let Some(Value::String(fixture)) = obj.get("fixture") { let path = spec_fixtures(ctx).join(fixture); - return match load(&path) { + return match prompty::load_with_options(&path, options) { Ok(agent) => Ok(agent_to_canonical(&agent)), Err(e) => Err(load_vector_error(e)), }; @@ -895,7 +918,7 @@ fn load_agent_from_input( let prompty_path = agent_dir.join("vector.prompty"); std::fs::write(&prompty_path, text).ok(); - let out = match load(&prompty_path) { + let out = match prompty::load_with_options(&prompty_path, options) { Ok(agent) => Ok(agent_to_canonical(&agent)), Err(e) => Err(load_vector_error(e)), }; diff --git a/runtime/rust/prompty/tests/model/vector_conformance_test.rs b/runtime/rust/prompty/tests/model/vector_conformance_test.rs index a1d46fb77..269207709 100644 --- a/runtime/rust/prompty/tests/model/vector_conformance_test.rs +++ b/runtime/rust/prompty/tests/model/vector_conformance_test.rs @@ -237,7 +237,7 @@ async fn test_vector_20_loadconformance_load_basic_load() { #[tokio::test] async fn test_vector_21_loadconformance_load_connection_types_load() { - let vector: Value = serde_json::from_str("{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}") + let vector: Value = serde_json::from_str("{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"trustedEndpoints\":[\"https://localhost:8080\"]},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } @@ -258,7 +258,7 @@ async fn test_vector_23_loadconformance_load_empty_frontmatter_body_only() { #[tokio::test] async fn test_vector_24_loadconformance_load_env_default() { - let vector: Value = serde_json::from_str("{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}") + let vector: Value = serde_json::from_str("{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\",\"apiKey\":\"literal-key\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; } @@ -425,49 +425,266 @@ async fn test_vector_47_loadconformance_load_tools_openapi_load() { } #[tokio::test] -async fn test_vector_48_memoryconformance_operate_clear_by_category_and_all() { +async fn test_vector_48_loadconformance_load_trusted_endpoint_builtin_allowed() { + let vector: Value = serde_json::from_str("{\"name\":\"trusted_endpoint_builtin_allowed\",\"description\":\"Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-builtin\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_49_loadconformance_load_trusted_endpoint_default_port_allowed() { + let vector: Value = serde_json::from_str("{\"name\":\"trusted_endpoint_default_port_allowed\",\"description\":\"An explicit default port normalizes to the built-in origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-default-port-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_50_loadconformance_load_trusted_endpoint_env_endpoint_allowed() { + let vector: Value = serde_json::from_str("{\"name\":\"trusted_endpoint_env_endpoint_allowed\",\"description\":\"An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_51_loadconformance_load_trusted_endpoint_env_list_allowed() { + let vector: Value = serde_json::from_str("{\"name\":\"trusted_endpoint_env_list_allowed\",\"description\":\"PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_52_loadconformance_load_trusted_endpoint_literal_key_allowed() { + let vector: Value = serde_json::from_str("{\"name\":\"trusted_endpoint_literal_key_allowed\",\"description\":\"A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_53_loadconformance_load_trusted_endpoint_option_allowed() { + let vector: Value = serde_json::from_str("{\"name\":\"trusted_endpoint_option_allowed\",\"description\":\"A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_54_loadconformance_load_trusted_endpoint_reference_skipped() { + let vector: Value = serde_json::from_str("{\"name\":\"trusted_endpoint_reference_skipped\",\"description\":\"kind: reference connections carry no endpoint and are not subject to the trust check\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_55_loadconformance_load_trusted_endpoint_userinfo_allowed() { + let vector: Value = serde_json::from_str("{\"name\":\"trusted_endpoint_userinfo_allowed\",\"description\":\"Userinfo is ignored; the destination host api.openai.com is trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-userinfo-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_56_loadconformance_load_untrusted_endpoint_ambient_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_ambient_rejected\",\"description\":\"A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec \u{a7}2.5.1)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_57_loadconformance_load_untrusted_endpoint_backslash_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_backslash_rejected\",\"description\":\"A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_58_loadconformance_load_untrusted_endpoint_blank_literal_key_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_blank_literal_key_rejected\",\"description\":\"a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-blank-key\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\" \"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_59_loadconformance_load_untrusted_endpoint_env_default_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_60_loadconformance_load_untrusted_endpoint_env_key_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_61_loadconformance_load_untrusted_endpoint_file_connection_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_62_loadconformance_load_untrusted_endpoint_fragment_before_at_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_fragment_before_at_rejected\",\"description\":\"A '#' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_63_loadconformance_load_untrusted_endpoint_ipv6_literal_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_ipv6_literal_rejected\",\"description\":\"An IPv6 literal is not a trusted origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_64_loadconformance_load_untrusted_endpoint_key_with_client_secret_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_65_loadconformance_load_untrusted_endpoint_mismatched_credential_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_66_loadconformance_load_untrusted_endpoint_no_key_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_67_loadconformance_load_untrusted_endpoint_oauth_literal_secret_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_oauth_literal_secret_rejected\",\"description\":\"kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_68_loadconformance_load_untrusted_endpoint_percent_encoded_host_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_percent_encoded_host_rejected\",\"description\":\"Hosts are compared without percent-decoding, so an encoded host never matches\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_69_loadconformance_load_untrusted_endpoint_port_mismatch_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_port_mismatch_rejected\",\"description\":\"A non-default port is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_70_loadconformance_load_untrusted_endpoint_query_before_at_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_query_before_at_rejected\",\"description\":\"A '?' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_71_loadconformance_load_untrusted_endpoint_scheme_mismatch_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_scheme_mismatch_rejected\",\"description\":\"Built-in origins are https; http on the same host is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_72_loadconformance_load_untrusted_endpoint_suffix_lookalike_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_suffix_lookalike_rejected\",\"description\":\"A trusted host as a label prefix of another domain is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_73_loadconformance_load_untrusted_endpoint_suffix_spoof_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_74_loadconformance_load_untrusted_endpoint_trailing_dot_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_trailing_dot_rejected\",\"description\":\"Hosts are compared exactly; a trailing-dot host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_75_loadconformance_load_untrusted_endpoint_unicode_homoglyph_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_unicode_homoglyph_rejected\",\"description\":\"A non-ASCII lookalike host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\u{430}pi.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_76_loadconformance_load_untrusted_endpoint_userinfo_lookalike_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_userinfo_lookalike_rejected\",\"description\":\"A trusted host in userinfo is not the destination host\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_77_loadconformance_load_untrusted_endpoint_wildcard_apex_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_78_loadconformance_load_untrusted_endpoint_wildcard_boundary_rejected() { + let vector: Value = serde_json::from_str("{\"name\":\"untrusted_endpoint_wildcard_boundary_rejected\",\"description\":\"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}") + .expect("failed to decode vector"); + vector_runner::vc_run_vector("LoadConformance", "load", vector, false, vc_seam()).await; +} + +#[tokio::test] +async fn test_vector_79_memoryconformance_operate_clear_by_category_and_all() { let vector: Value = serde_json::from_str("{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_49_memoryconformance_operate_format_core_and_recall_results() { +async fn test_vector_80_memoryconformance_operate_format_core_and_recall_results() { let vector: Value = serde_json::from_str("{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_50_memoryconformance_operate_recall_empty_query_and_limit() { +async fn test_vector_81_memoryconformance_operate_recall_empty_query_and_limit() { let vector: Value = serde_json::from_str("{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_51_memoryconformance_operate_recall_ranks_by_weighted_score() { +async fn test_vector_82_memoryconformance_operate_recall_ranks_by_weighted_score() { let vector: Value = serde_json::from_str("{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_52_memoryconformance_operate_recall_score_ties_preserve_insertion_order() { +async fn test_vector_83_memoryconformance_operate_recall_score_ties_preserve_insertion_order() { let vector: Value = serde_json::from_str("{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_53_memoryconformance_operate_recall_tags_and_core_boost() { +async fn test_vector_84_memoryconformance_operate_recall_tags_and_core_boost() { let vector: Value = serde_json::from_str("{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_54_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup() +async fn test_vector_85_memoryconformance_operate_recall_tokenization_punctuation_unicode_and_dedup() { let vector: Value = serde_json::from_str("{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\u{e9}\"]},{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\u{e9}\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}") .expect("failed to decode vector"); @@ -475,511 +692,511 @@ async fn test_vector_54_memoryconformance_operate_recall_tokenization_punctuatio } #[tokio::test] -async fn test_vector_55_memoryconformance_operate_remember_dedups_core_by_tags() { +async fn test_vector_86_memoryconformance_operate_remember_dedups_core_by_tags() { let vector: Value = serde_json::from_str("{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_56_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags() { +async fn test_vector_87_memoryconformance_operate_remember_dedups_core_missing_and_empty_tags() { let vector: Value = serde_json::from_str("{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_57_memoryconformance_operate_remember_evicts_archival_first() { +async fn test_vector_88_memoryconformance_operate_remember_evicts_archival_first() { let vector: Value = serde_json::from_str("{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_58_memoryconformance_operate_snapshot_port_round_trip() { +async fn test_vector_89_memoryconformance_operate_snapshot_port_round_trip() { let vector: Value = serde_json::from_str("{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_59_memoryconformance_operate_update_index_out_of_range_error() { +async fn test_vector_90_memoryconformance_operate_update_index_out_of_range_error() { let vector: Value = serde_json::from_str("{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("MemoryConformance", "operate", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_60_turnconformance_replay_max_iterations() { +async fn test_vector_91_turnconformance_replay_max_iterations() { let vector: Value = serde_json::from_str("{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_61_turnconformance_replay_no_tool() { +async fn test_vector_92_turnconformance_replay_no_tool() { let vector: Value = serde_json::from_str("{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_62_turnconformance_replay_permission_denied() { +async fn test_vector_93_turnconformance_replay_permission_denied() { let vector: Value = serde_json::from_str("{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_63_turnconformance_replay_tool_failure() { +async fn test_vector_94_turnconformance_replay_tool_failure() { let vector: Value = serde_json::from_str("{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_64_turnconformance_replay_tool_success() { +async fn test_vector_95_turnconformance_replay_tool_success() { let vector: Value = serde_json::from_str("{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "replay", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_65_turnconformance_run_assistant_tool_calls_metadata() { +async fn test_vector_96_turnconformance_run_assistant_tool_calls_metadata() { let vector: Value = serde_json::from_str("{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_66_turnconformance_run_async_tool_function() { +async fn test_vector_97_turnconformance_run_async_tool_function() { let vector: Value = serde_json::from_str("{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_67_turnconformance_run_bindings_injected() { +async fn test_vector_98_turnconformance_run_bindings_injected() { let vector: Value = serde_json::from_str("{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_68_turnconformance_run_cancellation_before_llm() { +async fn test_vector_99_turnconformance_run_cancellation_before_llm() { let vector: Value = serde_json::from_str("{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_69_turnconformance_run_cancellation_between_iterations() { +async fn test_vector_100_turnconformance_run_cancellation_between_iterations() { let vector: Value = serde_json::from_str("{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_70_turnconformance_run_cancellation_between_tools() { +async fn test_vector_101_turnconformance_run_cancellation_between_tools() { let vector: Value = serde_json::from_str("{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_71_turnconformance_run_context_no_trim_when_fits() { +async fn test_vector_102_turnconformance_run_context_no_trim_when_fits() { let vector: Value = serde_json::from_str("{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_72_turnconformance_run_context_preserves_system_messages() { +async fn test_vector_103_turnconformance_run_context_preserves_system_messages() { let vector: Value = serde_json::from_str("{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_73_turnconformance_run_context_trim_basic() { +async fn test_vector_104_turnconformance_run_context_trim_basic() { let vector: Value = serde_json::from_str("{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_74_turnconformance_run_empty_tool_result() { +async fn test_vector_105_turnconformance_run_empty_tool_result() { let vector: Value = serde_json::from_str("{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_75_turnconformance_run_events_basic_tool_loop() { +async fn test_vector_106_turnconformance_run_events_basic_tool_loop() { let vector: Value = serde_json::from_str("{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_76_turnconformance_run_events_error_logged() { +async fn test_vector_107_turnconformance_run_events_error_logged() { let vector: Value = serde_json::from_str("{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_77_turnconformance_run_events_no_tools() { +async fn test_vector_108_turnconformance_run_events_no_tools() { let vector: Value = serde_json::from_str("{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_78_turnconformance_run_guardrail_all_pass() { +async fn test_vector_109_turnconformance_run_guardrail_all_pass() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_79_turnconformance_run_guardrail_input_deny() { +async fn test_vector_110_turnconformance_run_guardrail_input_deny() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_80_turnconformance_run_guardrail_output_deny() { +async fn test_vector_111_turnconformance_run_guardrail_output_deny() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_81_turnconformance_run_guardrail_tool_deny() { +async fn test_vector_112_turnconformance_run_guardrail_tool_deny() { let vector: Value = serde_json::from_str("{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_82_turnconformance_run_max_iterations_exceeded() { +async fn test_vector_113_turnconformance_run_max_iterations_exceeded() { let vector: Value = serde_json::from_str("{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_83_turnconformance_run_multi_turn_tool_calls() { +async fn test_vector_114_turnconformance_run_multi_turn_tool_calls() { let vector: Value = serde_json::from_str("{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_84_turnconformance_run_multiple_tool_calls_single_turn() { +async fn test_vector_115_turnconformance_run_multiple_tool_calls_single_turn() { let vector: Value = serde_json::from_str("{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_85_turnconformance_run_no_tool_calls() { +async fn test_vector_116_turnconformance_run_no_tool_calls() { let vector: Value = serde_json::from_str("{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_86_turnconformance_run_parallel_tools_basic() { +async fn test_vector_117_turnconformance_run_parallel_tools_basic() { let vector: Value = serde_json::from_str("{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_87_turnconformance_run_parallel_tools_with_guardrail_deny() { +async fn test_vector_118_turnconformance_run_parallel_tools_with_guardrail_deny() { let vector: Value = serde_json::from_str("{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_88_turnconformance_run_parallel_tools_with_tool_error() { +async fn test_vector_119_turnconformance_run_parallel_tools_with_tool_error() { let vector: Value = serde_json::from_str("{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\u{a7}13.6 Parallel Tools + tool error \u{2014} 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_89_turnconformance_run_single_tool_call() { +async fn test_vector_120_turnconformance_run_single_tool_call() { let vector: Value = serde_json::from_str("{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_90_turnconformance_run_steering_inject_message() { +async fn test_vector_121_turnconformance_run_steering_inject_message() { let vector: Value = serde_json::from_str("{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_91_turnconformance_run_steering_multiple_messages() { +async fn test_vector_122_turnconformance_run_steering_multiple_messages() { let vector: Value = serde_json::from_str("{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_92_turnconformance_run_tool_not_registered_error() { +async fn test_vector_123_turnconformance_run_tool_not_registered_error() { let vector: Value = serde_json::from_str("{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_93_turnconformance_run_tool_result_message_format() { +async fn test_vector_124_turnconformance_run_tool_result_message_format() { let vector: Value = serde_json::from_str("{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "run", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_94_turnconformance_runturn_cancel_before_context() { +async fn test_vector_125_turnconformance_runturn_cancel_before_context() { let vector: Value = serde_json::from_str("{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_95_turnconformance_runturn_delegated_provider_state() { +async fn test_vector_126_turnconformance_runturn_delegated_provider_state() { let vector: Value = serde_json::from_str("{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_96_turnconformance_runturn_final_output() { +async fn test_vector_127_turnconformance_runturn_final_output() { let vector: Value = serde_json::from_str("{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_97_turnconformance_runturn_memory_context_is_model_visible() { +async fn test_vector_128_turnconformance_runturn_memory_context_is_model_visible() { let vector: Value = serde_json::from_str("{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_98_turnconformance_runturn_ordered_tool_round() { +async fn test_vector_129_turnconformance_runturn_ordered_tool_round() { let vector: Value = serde_json::from_str("{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_99_turnconformance_runturn_permission_denial_is_model_visible() { +async fn test_vector_130_turnconformance_runturn_permission_denial_is_model_visible() { let vector: Value = serde_json::from_str("{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("TurnConformance", "runTurn", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_100_wireconformance_torequest_anthropic_image_format() { +async fn test_vector_131_wireconformance_torequest_anthropic_image_format() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_101_wireconformance_torequest_anthropic_max_tokens_required() { +async fn test_vector_132_wireconformance_torequest_anthropic_max_tokens_required() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_102_wireconformance_torequest_anthropic_options() { +async fn test_vector_133_wireconformance_torequest_anthropic_options() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_103_wireconformance_torequest_anthropic_system_separate() { +async fn test_vector_134_wireconformance_torequest_anthropic_system_separate() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_104_wireconformance_torequest_anthropic_tool_wire() { +async fn test_vector_135_wireconformance_torequest_anthropic_tool_wire() { let vector: Value = serde_json::from_str("{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_105_wireconformance_torequest_chat_audio_mp3() { +async fn test_vector_136_wireconformance_torequest_chat_audio_mp3() { let vector: Value = serde_json::from_str("{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_106_wireconformance_torequest_chat_audio_part() { +async fn test_vector_137_wireconformance_torequest_chat_audio_part() { let vector: Value = serde_json::from_str("{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_107_wireconformance_torequest_chat_image_base64() { +async fn test_vector_138_wireconformance_torequest_chat_image_base64() { let vector: Value = serde_json::from_str("{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_108_wireconformance_torequest_chat_image_part() { +async fn test_vector_139_wireconformance_torequest_chat_image_part() { let vector: Value = serde_json::from_str("{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_109_wireconformance_torequest_chat_multipart_content() { +async fn test_vector_140_wireconformance_torequest_chat_multipart_content() { let vector: Value = serde_json::from_str("{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_110_wireconformance_torequest_chat_simple() { +async fn test_vector_141_wireconformance_torequest_chat_simple() { let vector: Value = serde_json::from_str("{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_111_wireconformance_torequest_chat_single_text_optimized() { +async fn test_vector_142_wireconformance_torequest_chat_single_text_optimized() { let vector: Value = serde_json::from_str("{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_112_wireconformance_torequest_chat_with_options() { +async fn test_vector_143_wireconformance_torequest_chat_with_options() { let vector: Value = serde_json::from_str("{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_113_wireconformance_torequest_embedding_wire() { +async fn test_vector_144_wireconformance_torequest_embedding_wire() { let vector: Value = serde_json::from_str("{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_114_wireconformance_torequest_image_wire() { +async fn test_vector_145_wireconformance_torequest_image_wire() { let vector: Value = serde_json::from_str("{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_115_wireconformance_torequest_kind_to_json_type_mapping() { +async fn test_vector_146_wireconformance_torequest_kind_to_json_type_mapping() { let vector: Value = serde_json::from_str("{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_116_wireconformance_torequest_options_additional_properties() { +async fn test_vector_147_wireconformance_torequest_options_additional_properties() { let vector: Value = serde_json::from_str("{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_117_wireconformance_torequest_options_max_completion_tokens() { +async fn test_vector_148_wireconformance_torequest_options_max_completion_tokens() { let vector: Value = serde_json::from_str("{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_118_wireconformance_torequest_options_stop_sequences() { +async fn test_vector_149_wireconformance_torequest_options_stop_sequences() { let vector: Value = serde_json::from_str("{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_119_wireconformance_torequest_responses_simple() { +async fn test_vector_150_wireconformance_torequest_responses_simple() { let vector: Value = serde_json::from_str("{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_120_wireconformance_torequest_responses_structured_output() { +async fn test_vector_151_wireconformance_torequest_responses_structured_output() { let vector: Value = serde_json::from_str("{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_121_wireconformance_torequest_responses_with_tools() { +async fn test_vector_152_wireconformance_torequest_responses_with_tools() { let vector: Value = serde_json::from_str("{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_122_wireconformance_torequest_structured_output() { +async fn test_vector_153_wireconformance_torequest_structured_output() { let vector: Value = serde_json::from_str("{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_123_wireconformance_torequest_structured_output_nested_optional() { +async fn test_vector_154_wireconformance_torequest_structured_output_nested_optional() { let vector: Value = serde_json::from_str("{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_124_wireconformance_torequest_tools_bindings_stripped() { +async fn test_vector_155_wireconformance_torequest_tools_bindings_stripped() { let vector: Value = serde_json::from_str("{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_125_wireconformance_torequest_tools_function_wire() { +async fn test_vector_156_wireconformance_torequest_tools_function_wire() { let vector: Value = serde_json::from_str("{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_126_wireconformance_torequest_tools_null_when_empty() { +async fn test_vector_157_wireconformance_torequest_tools_null_when_empty() { let vector: Value = serde_json::from_str("{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; } #[tokio::test] -async fn test_vector_127_wireconformance_torequest_tools_strict_mode() { +async fn test_vector_158_wireconformance_torequest_tools_strict_mode() { let vector: Value = serde_json::from_str("{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}") .expect("failed to decode vector"); vector_runner::vc_run_vector("WireConformance", "toRequest", vector, false, vc_seam()).await; diff --git a/runtime/swift/README.md b/runtime/swift/README.md index 3a0eb49df..ca27dba24 100644 --- a/runtime/swift/README.md +++ b/runtime/swift/README.md @@ -55,6 +55,36 @@ let messages = try await Pipeline.prepare(agent, inputs: inputs) // render + pa let raw = try await Pipeline.run(agent, messages: messages) // execute + process ``` +### Endpoint trust + +A `.prompty` file is untrusted input. If a connection uses ambient credentials +(an `${env:}` API key, a bearer token, a managed identity), the loader refuses a +literal `endpoint` or `tokenUrl` that the host hasn't approved. It throws +`LoadError.untrustedEndpoint` (spec §2.5.1). By default the loader trusts these +origins: + +- `https://api.openai.com` +- `https://api.anthropic.com` +- `https://*.openai.azure.com` +- `https://*.services.ai.azure.com` +- `https://*.cognitiveservices.azure.com` +- `https://login.microsoftonline.com` + +To trust other origins, pass them as a load option or set them in the +environment: + +```swift +let agent = try Loader.load( + path: "local.prompty", + options: LoadOptions(trustedEndpoints: ["http://localhost:11434", "https://*.contoso.com"]) +) +``` + +`PROMPTY_TRUSTED_ENDPOINTS` is a comma-separated list of origins and is merged +with the option. `*` trusts everything. The check passes without any trust entry +when the endpoint comes from `${env:VAR}` (host-controlled) or when the file +carries its own literal key. + ### Tool calls When a prompt declares tools, the host drives the loop. Read the calls, then ask diff --git a/runtime/swift/prompty-model/Info.plist b/runtime/swift/prompty-model/Info.plist new file mode 100644 index 000000000..c33b7789a --- /dev/null +++ b/runtime/swift/prompty-model/Info.plist @@ -0,0 +1,16 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleIdentifier + prompty-model.PromptyModel.resources + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + PromptyModel_PromptyModel + CFBundlePackageType + BNDL + + diff --git a/runtime/swift/prompty-model/Sources/PromptyModel/EndpointTrust.swift b/runtime/swift/prompty-model/Sources/PromptyModel/EndpointTrust.swift new file mode 100644 index 000000000..7b95ea5f8 --- /dev/null +++ b/runtime/swift/prompty-model/Sources/PromptyModel/EndpointTrust.swift @@ -0,0 +1,195 @@ +import Foundation + +/// Endpoint trust — keep ambient credentials on host-approved destinations (spec §2.5.1). +/// +/// A `.prompty` file is untrusted input. When its connection carries no +/// file-supplied credential, executors fall back to ambient credentials +/// (Azure identity tokens, `*_API_KEY` env vars). This decides whether the +/// file-chosen destination may receive them. +public enum EndpointTrust { + /// Comma-separated list of extra trusted origins supplied by the host environment. + public static let trustedEndpointsEnv = "PROMPTY_TRUSTED_ENDPOINTS" + + /// First-party provider origins trusted without host configuration. + public static let builtinTrustedEndpoints: [String] = [ + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com", + ] + + private static let defaultPorts: [String: Int] = ["http": 80, "https": 443] + private static let credentialKeys: [String: String] = ["key": "apiKey"] + private static let destinationKeys = ["endpoint", "tokenUrl"] + + /// Return `true` if `url`'s origin matches a built-in, host, or env trusted origin. + public static func isTrusted(_ url: String, trustedEndpoints: [String] = []) -> Bool { + let entries = trustedEntries(trustedEndpoints) + if entries.contains("*") { return true } + guard let target = origin(url) else { return false } + for entry in entries { + guard let allowed = origin(entry), allowed.scheme == target.scheme, allowed.port == target.port else { + continue + } + if allowed.host.hasPrefix("*.") { + if target.host.hasSuffix(String(allowed.host.dropFirst())) { return true } + } else if allowed.host == target.host { + return true + } + } + return false + } + + /// Throw `untrusted_endpoint` if ambient credentials could reach a file-chosen, untrusted destination. + /// + /// `connection` is the RAW (unresolved) `model.connection` mapping. + public static func checkConnection(_ connection: Any?, trustedEndpoints: [String] = []) throws { + guard let connection = connection as? [String: Any] else { return } + if connection["kind"] as? String == "reference" { return } + if fileSuppliedCredential(connection) { return } + for key in destinationKeys { + guard let destination = fileControlledDestination(connection[key]), + !isTrusted(destination, trustedEndpoints: trustedEndpoints) + else { continue } + throw PromptyLoadError( + kind: "untrusted_endpoint", + message: "Connection \(key) '\(destination)' is not trusted to receive ambient credentials. " + + "Set the \(key) via ${env:VAR}, add its origin to trustedEndpoints or " + + "\(trustedEndpointsEnv), or use 'kind: reference'.") + } + } + + /// Return the unresolved `model.connection`, following whole-value `${file:}` references. + /// + /// `resolveFile` receives the reference path and returns the loaded content, or + /// `nil` when the file does not exist (resolution reports that later). + public static func rawConnection( + _ data: [String: Any], resolveFile: (String) throws -> Any? + ) rethrows -> Any? { + func follow(_ value: Any?) throws -> Any? { + if let string = value as? String, isReference(string) { + let inner = String(string.dropFirst(2).dropLast()) + if let colon = inner.firstIndex(of: ":"), inner[.. String? { + if promptyEnvironmentOverrides.keys.contains(name) { + return promptyEnvironmentOverrides[name] ?? nil + } + return ProcessInfo.processInfo.environment[name] + } + + private static func trustedEntries(_ trustedEndpoints: [String]) -> [String] { + var entries = builtinTrustedEndpoints + trustedEndpoints + entries.append( + contentsOf: (environmentValue(trustedEndpointsEnv) ?? "").split(separator: ",", omittingEmptySubsequences: false) + .map(String.init)) + return entries.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty } + } + + private static func isReference(_ value: String) -> Bool { + value.hasPrefix("${") && value.hasSuffix("}") + } + + private static func fileSuppliedCredential(_ connection: [String: Any]) -> Bool { + // Only the credential field the declared kind actually consumes counts; any other kind is ambient. + guard let kind = connection["kind"] as? String, let key = credentialKeys[kind], + let value = connection[key] as? String + else { return false } + // Blank keys count as absent: executors that trim fall back to ambient credentials. + return !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && !isReference(value) + } + + /// Return the effective destination if the file chose it, else `nil`. + private static func fileControlledDestination(_ value: Any?) -> String? { + guard let value, !(value is NSNull) else { return nil } + guard let string = value as? String else { return "\(value)" } + if isReference(string) { + let inner = String(string.dropFirst(2).dropLast()) + let parts = inner.split(separator: ":", maxSplits: 1, omittingEmptySubsequences: false) + if parts[0].lowercased() == "env" { + let rest = parts.count > 1 ? String(parts[1]) : "" + let pieces = rest.split(separator: ":", maxSplits: 1, omittingEmptySubsequences: false) + let name = String(pieces[0]) + if environmentValue(name) != nil { return nil } + let fallback = pieces.count > 1 ? String(pieces[1]) : "" + return fallback.isEmpty ? nil : fallback + } + } + return string + } + + struct Origin: Equatable { + let scheme: String + let host: String + let port: Int + } + + /// Return `(scheme, host, port)` for an absolute http(s) URL, else `nil`. + /// + /// Mirrors Python's `urllib.parse.urlsplit` so every runtime classifies the same strings alike. + static func origin(_ url: String) -> Origin? { + // HTTP clients treat '\' as '/', so parsers disagree on the host; reject outright. + if url.contains("\\") { return nil } + let trimmed = url.trimmingCharacters(in: .whitespacesAndNewlines) + let text = String(String.UnicodeScalarView(trimmed.unicodeScalars.filter { !"\t\r\n".unicodeScalars.contains($0) })) + guard let colon = text.firstIndex(of: ":") else { return nil } + let rawScheme = text[.. Agent { return try Agent.load(data) } -public func loadPromptyContent(_ contents: String, parentDir: URL, allowedRoots: [URL]) throws -> Agent { +/// Enforce endpoint trust (spec §2.5.1) on the RAW frontmatter, before references resolve. +/// +/// Whole-value `${file:}` references for `model` and `model.connection` are followed +/// with the same containment as reference resolution. +public func checkEndpointTrust( + _ data: [String: Any], parentDir: URL, allowedRoots: [URL], trustedEndpoints: [String] = [] +) throws { + let connection = try EndpointTrust.rawConnection(data) { reference in + guard FileManager.default.fileExists(atPath: relativeFileURL(reference, base: parentDir).path) else { + return nil + } + return try resolveFile(reference, parentDir: parentDir, allowedRoots: allowedRoots) + } + try EndpointTrust.checkConnection(connection, trustedEndpoints: trustedEndpoints) +} + +public func loadPromptyContent( + _ contents: String, parentDir: URL, allowedRoots: [URL], trustedEndpoints: [String] = [] +) throws -> Agent { var data = try parseFrontmatter(contents) + try checkEndpointTrust(data, parentDir: parentDir, allowedRoots: allowedRoots, trustedEndpoints: trustedEndpoints) try resolveReferences(&data, parentDir: parentDir, allowedRoots: allowedRoots) return try buildAgentFromData(data) } -public func loadPromptyFile(_ path: URL) throws -> Agent { +public func loadPromptyFile(_ path: URL, trustedEndpoints: [String] = []) throws -> Agent { guard FileManager.default.fileExists(atPath: path.path) else { throw PromptyLoadError(kind: "not_found", message: "FileNotFoundError: prompty file not found: '\(path.path)'") } let text = (try String(contentsOf: path, encoding: .utf8)).replacingOccurrences(of: "\r\n", with: "\n") let dir = canonicalURL(path.deletingLastPathComponent()) - return try loadPromptyContent(text, parentDir: dir, allowedRoots: [dir]) + return try loadPromptyContent(text, parentDir: dir, allowedRoots: [dir], trustedEndpoints: trustedEndpoints) } public func validateInputs(agent: Agent, provided: [String: Any]) throws -> [String: Any] { diff --git a/runtime/swift/prompty-model/Sources/PromptyModel/vector_conformance.swift b/runtime/swift/prompty-model/Sources/PromptyModel/vector_conformance.swift index df1feb020..76f12e1c0 100644 --- a/runtime/swift/prompty-model/Sources/PromptyModel/vector_conformance.swift +++ b/runtime/swift/prompty-model/Sources/PromptyModel/vector_conformance.swift @@ -276,7 +276,7 @@ public func runLoadConformanceConformance(_ seam: S) async t } // vector: connection_types_load do { - let inputData = "{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}}".data(using: .utf8)! + let inputData = "{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"trustedEndpoints\":[\"https://localhost:8080\"]}".data(using: .utf8)! guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { throw SeamConformanceError("connection_types_load: failed to parse embedded vector input") } @@ -318,7 +318,7 @@ public func runLoadConformanceConformance(_ seam: S) async t } // vector: env_default do { - let inputData = "{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}}".data(using: .utf8)! + let inputData = "{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\",\"apiKey\":\"literal-key\"}}},\"env\":{}}".data(using: .utf8)! guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { throw SeamConformanceError("env_default: failed to parse embedded vector input") } @@ -670,4 +670,507 @@ public func runLoadConformanceConformance(_ seam: S) async t throw SeamConformanceError("tools_openapi_load misrouted") } } + // vector: trusted_endpoint_builtin_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-builtin\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_builtin_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_builtin_allowed misrouted") + } + } + // vector: trusted_endpoint_default_port_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-default-port-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_default_port_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_default_port_allowed misrouted") + } + } + // vector: trusted_endpoint_env_endpoint_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_env_endpoint_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_env_endpoint_allowed misrouted") + } + } + // vector: trusted_endpoint_env_list_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_env_list_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_env_list_allowed misrouted") + } + } + // vector: trusted_endpoint_literal_key_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_literal_key_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_literal_key_allowed misrouted") + } + } + // vector: trusted_endpoint_option_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_option_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_option_allowed misrouted") + } + } + // vector: trusted_endpoint_reference_skipped + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_reference_skipped: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_reference_skipped misrouted") + } + } + // vector: trusted_endpoint_userinfo_allowed + do { + let inputData = "{\"frontmatter\":{\"name\":\"trusted-userinfo-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("trusted_endpoint_userinfo_allowed: failed to parse embedded vector input") + } + let source = input["source"] as! Any + let actual = try await seam.load(source: source) + let expectedData = "{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}".data(using: .utf8)! + let expected = try JSONSerialization.jsonObject(with: expectedData, options: [.fragmentsAllowed]) + guard try seamConformanceCanonicalJSON(try actual.save()) == seamConformanceCanonicalJSON(expected) else { + throw SeamConformanceError("trusted_endpoint_userinfo_allowed misrouted") + } + } + // vector: untrusted_endpoint_ambient_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_ambient_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_ambient_rejected: expected an error") + } + } + // vector: untrusted_endpoint_backslash_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_backslash_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_backslash_rejected: expected an error") + } + } + // vector: untrusted_endpoint_blank_literal_key_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-blank-key\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\" \"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_blank_literal_key_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_blank_literal_key_rejected: expected an error") + } + } + // vector: untrusted_endpoint_env_default_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_env_default_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_env_default_rejected: expected an error") + } + } + // vector: untrusted_endpoint_env_key_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_env_key_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_env_key_rejected: expected an error") + } + } + // vector: untrusted_endpoint_file_connection_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_file_connection_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_file_connection_rejected: expected an error") + } + } + // vector: untrusted_endpoint_fragment_before_at_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_fragment_before_at_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_fragment_before_at_rejected: expected an error") + } + } + // vector: untrusted_endpoint_ipv6_literal_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_ipv6_literal_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_ipv6_literal_rejected: expected an error") + } + } + // vector: untrusted_endpoint_key_with_client_secret_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_key_with_client_secret_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_key_with_client_secret_rejected: expected an error") + } + } + // vector: untrusted_endpoint_mismatched_credential_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_mismatched_credential_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_mismatched_credential_rejected: expected an error") + } + } + // vector: untrusted_endpoint_no_key_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_no_key_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_no_key_rejected: expected an error") + } + } + // vector: untrusted_endpoint_oauth_literal_secret_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_oauth_literal_secret_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_oauth_literal_secret_rejected: expected an error") + } + } + // vector: untrusted_endpoint_percent_encoded_host_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_percent_encoded_host_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_percent_encoded_host_rejected: expected an error") + } + } + // vector: untrusted_endpoint_port_mismatch_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_port_mismatch_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_port_mismatch_rejected: expected an error") + } + } + // vector: untrusted_endpoint_query_before_at_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_query_before_at_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_query_before_at_rejected: expected an error") + } + } + // vector: untrusted_endpoint_scheme_mismatch_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_scheme_mismatch_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_scheme_mismatch_rejected: expected an error") + } + } + // vector: untrusted_endpoint_suffix_lookalike_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_suffix_lookalike_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_suffix_lookalike_rejected: expected an error") + } + } + // vector: untrusted_endpoint_suffix_spoof_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_suffix_spoof_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_suffix_spoof_rejected: expected an error") + } + } + // vector: untrusted_endpoint_trailing_dot_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_trailing_dot_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_trailing_dot_rejected: expected an error") + } + } + // vector: untrusted_endpoint_unicode_homoglyph_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\u{430}pi.openai.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_unicode_homoglyph_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_unicode_homoglyph_rejected: expected an error") + } + } + // vector: untrusted_endpoint_userinfo_lookalike_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_userinfo_lookalike_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_userinfo_lookalike_rejected: expected an error") + } + } + // vector: untrusted_endpoint_wildcard_apex_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_wildcard_apex_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_wildcard_apex_rejected: expected an error") + } + } + // vector: untrusted_endpoint_wildcard_boundary_rejected + do { + let inputData = "{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}}".data(using: .utf8)! + guard let input = try JSONSerialization.jsonObject(with: inputData) as? [String: Any] else { + throw SeamConformanceError("untrusted_endpoint_wildcard_boundary_rejected: failed to parse embedded vector input") + } + let source = input["source"] as! Any + var caught: Error? = nil + do { + _ = try await seam.load(source: source) + } catch { + caught = error + } + guard let caughtError = caught else { + throw SeamConformanceError("untrusted_endpoint_wildcard_boundary_rejected: expected an error") + } + } } diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift index 386230829..263b3a64f 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorAdapters.swift @@ -313,8 +313,10 @@ enum VectorAdapters { static func loadInvoke(_ input: Any?, _ context: VectorContext) throws -> Any? { let object = input as? [String: Any] ?? [:] - let restore = applyEnv(object["env"]) + let env = object["env"] as? [String: Any] ?? [:] + let restore = applyEnv(env, unset: env[EndpointTrust.trustedEndpointsEnv] == nil ? [EndpointTrust.trustedEndpointsEnv] : []) defer { restore() } + let trusted = (object["trustedEndpoints"] as? [Any] ?? []).compactMap { $0 as? String } // Input-validation vectors carry inputs alongside frontmatter at the top level; // full-load vectors nest inputs inside the frontmatter. @@ -330,11 +332,11 @@ enum VectorAdapters { do { let agent: Agent if let fixture = object["fixture"] as? String, !fixture.isEmpty { - agent = try loadPromptyFile(findSpecFixtures(context.baseDir).appendingPathComponent(fixture)) + agent = try loadPromptyFile(findSpecFixtures(context.baseDir).appendingPathComponent(fixture), trustedEndpoints: trusted) } else if let raw = object["frontmatter_raw"] as? String, !raw.isEmpty { - agent = try loadFromRaw(raw) + agent = try loadFromRaw(raw, trustedEndpoints: trusted) } else if object["frontmatter"] is [String: Any] { - agent = try materializeAndLoad(object) + agent = try materializeAndLoad(object, trustedEndpoints: trusted) } else { return ["error": ""] } @@ -361,13 +363,15 @@ enum VectorAdapters { return root } - static func loadFromRaw(_ raw: String) throws -> Agent { + static func loadFromRaw(_ raw: String, trustedEndpoints: [String] = []) throws -> Agent { let dir = try scratchDir("prompty-load-") defer { try? FileManager.default.removeItem(at: dir) } - return try loadPromptyContent(raw.replacingOccurrences(of: "\r\n", with: "\n"), parentDir: dir, allowedRoots: [dir]) + return try loadPromptyContent( + raw.replacingOccurrences(of: "\r\n", with: "\n"), parentDir: dir, allowedRoots: [dir], + trustedEndpoints: trustedEndpoints) } - static func materializeAndLoad(_ input: [String: Any]) throws -> Agent { + static func materializeAndLoad(_ input: [String: Any], trustedEndpoints: [String] = []) throws -> Agent { let tempBase = try scratchDir("prompty-load-") defer { try? FileManager.default.removeItem(at: tempBase) } var agentDir = tempBase @@ -382,6 +386,7 @@ enum VectorAdapters { } var frontmatter = input["frontmatter"] as? [String: Any] ?? [:] let root = agentDir.standardizedFileURL + try checkEndpointTrust(frontmatter, parentDir: agentDir, allowedRoots: [root], trustedEndpoints: trustedEndpoints) try resolveReferences(&frontmatter, parentDir: agentDir, allowedRoots: [root]) return try buildAgentFromData(frontmatter) } @@ -455,16 +460,31 @@ enum VectorAdapters { return payload } - static func applyEnv(_ value: Any?) -> () -> Void { + /// Apply a vector's `env` (and force-unset `unset`) through `promptyEnvironmentOverrides`, + /// returning a closure that restores the prior override state exactly. + static func applyEnv(_ value: Any?, unset: [String] = []) -> () -> Void { let env = value as? [String: Any] ?? [:] - var saved: [String: String?] = [:] + var saved: [String: String??] = [:] + func save(_ key: String) { + if !saved.keys.contains(key) { + saved[key] = promptyEnvironmentOverrides.keys.contains(key) ? .some(promptyEnvironmentOverrides[key] ?? nil) : .none + } + } for (key, raw) in env { - saved[key] = promptyEnvironmentOverrides[key] ?? nil + save(key) promptyEnvironmentOverrides[key] = "\(raw)" } + for key in unset { + save(key) + promptyEnvironmentOverrides[key] = .some(nil) + } return { for (key, previous) in saved { - promptyEnvironmentOverrides[key] = previous + if let previous { + promptyEnvironmentOverrides[key] = previous + } else { + promptyEnvironmentOverrides.removeValue(forKey: key) + } } } } diff --git a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift index b5cc59dc1..da3c7a341 100644 --- a/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift +++ b/runtime/swift/prompty-model/Tests/PromptyModelTests/VectorConformanceTests.swift @@ -245,7 +245,7 @@ final class VectorConformanceTests: XCTestCase { } func testVector21LoadConformanceLoadConnectionTypesLoad() async throws { - guard let vectorData = "{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}}},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}".data(using: .utf8), + guard let vectorData = "{\"name\":\"connection_types_load\",\"description\":\"All connection kinds (key, reference, remote, anonymous) load correctly\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"connection-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"trustedEndpoints\":[\"https://localhost:8080\"]},\"expected\":{\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"anonymous\",\"endpoint\":\"https://localhost:8080\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { XCTFail("failed to parse embedded vector") @@ -275,7 +275,7 @@ final class VectorConformanceTests: XCTestCase { } func testVector24LoadConformanceLoadEnvDefault() async throws { - guard let vectorData = "{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}".data(using: .utf8), + guard let vectorData = "{\"name\":\"env_default\",\"description\":\"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"env-default-test\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"${env:MISSING_VAR:fallback_value}\",\"apiKey\":\"literal-key\"}}},\"env\":{}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"fallback_value\"}}},\"operation\":\"load\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { XCTFail("failed to parse embedded vector") @@ -514,7 +514,317 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) } - func testVector48MemoryConformanceOperateClearByCategoryAndAll() async throws { + func testVector48LoadConformanceLoadTrustedEndpointBuiltinAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_builtin_allowed\",\"description\":\"Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-builtin\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://myres.services.ai.azure.com/api/projects/p\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector49LoadConformanceLoadTrustedEndpointDefaultPortAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_default_port_allowed\",\"description\":\"An explicit default port normalizes to the built-in origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-default-port-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://API.OpenAI.com:443/v1\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector50LoadConformanceLoadTrustedEndpointEnvEndpointAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_env_endpoint_allowed\",\"description\":\"An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-endpoint\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:VECTOR_FOUNDRY_ENDPOINT}\"}}},\"env\":{\"VECTOR_FOUNDRY_ENDPOINT\":\"https://host-chosen.example.com\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://host-chosen.example.com\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector51LoadConformanceLoadTrustedEndpointEnvListAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_env_list_allowed\",\"description\":\"PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-env-list\",\"model\":{\"id\":\"llama3\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"env\":{\"PROMPTY_TRUSTED_ENDPOINTS\":\"https://other.example.com, http://localhost:11434\"}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://localhost:11434/v1\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector52LoadConformanceLoadTrustedEndpointLiteralKeyAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_literal_key_allowed\",\"description\":\"A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-literal-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\",\"apiKey\":\"file-supplied-key\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://anywhere.example.com\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector53LoadConformanceLoadTrustedEndpointOptionAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_option_allowed\",\"description\":\"A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-option\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"trustedEndpoints\":[\"HTTPS://PROXY.Example.com\"]},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://proxy.example.com:443/openai/v1\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector54LoadConformanceLoadTrustedEndpointReferenceSkipped() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_reference_skipped\",\"description\":\"kind: reference connections carry no endpoint and are not subject to the trust check\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-reference\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"reference\",\"name\":\"my-connection\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector55LoadConformanceLoadTrustedEndpointUserinfoAllowed() async throws { + guard let vectorData = "{\"name\":\"trusted_endpoint_userinfo_allowed\",\"description\":\"Userinfo is ignored; the destination host api.openai.com is trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"trusted-userinfo-allowed\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}}},\"expected\":{\"model\":{\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://user@api.openai.com/v1\"}}},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector56LoadConformanceLoadUntrustedEndpointAmbientRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_ambient_rejected\",\"description\":\"A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec \u{a7}2.5.1)\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ambient\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com/api/projects/p\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector57LoadConformanceLoadUntrustedEndpointBackslashRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_backslash_rejected\",\"description\":\"A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-backslash\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com\\\\@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector58LoadConformanceLoadUntrustedEndpointBlankLiteralKeyRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_blank_literal_key_rejected\",\"description\":\"a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-blank-key\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"foundry\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\" \"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector59LoadConformanceLoadUntrustedEndpointEnvDefaultRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_env_default_rejected\",\"description\":\"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-default\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"${env:NONEXISTENT:https://attacker.example.com}\"}}},\"env\":{}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector60LoadConformanceLoadUntrustedEndpointEnvKeyRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_env_key_rejected\",\"description\":\"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-env-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"${env:VECTOR_HOST_KEY}\"}}},\"env\":{\"VECTOR_HOST_KEY\":\"host-secret\"}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector61LoadConformanceLoadUntrustedEndpointFileConnectionRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_file_connection_rejected\",\"description\":\"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-file-conn\",\"model\":{\"id\":\"gpt-4\",\"connection\":\"${file:conn.json}\"}},\"files\":{\"conn.json\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\"}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector62LoadConformanceLoadUntrustedEndpointFragmentBeforeAtRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_fragment_before_at_rejected\",\"description\":\"A '#' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-fragment-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com#@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector63LoadConformanceLoadUntrustedEndpointIpv6LiteralRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_ipv6_literal_rejected\",\"description\":\"An IPv6 literal is not a trusted origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-ipv6-literal\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://[::1]:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector64LoadConformanceLoadUntrustedEndpointKeyWithClientSecretRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_key_with_client_secret_rejected\",\"description\":\"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-key-client-secret\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"https://attacker.example.com\",\"clientSecret\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector65LoadConformanceLoadUntrustedEndpointMismatchedCredentialRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_mismatched_credential_rejected\",\"description\":\"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-mismatched-credential\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://attacker.example.com\",\"apiKey\":\"junk\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector66LoadConformanceLoadUntrustedEndpointNoKeyRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_no_key_rejected\",\"description\":\"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-no-key\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"key\",\"endpoint\":\"http://127.0.0.1:8080\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector67LoadConformanceLoadUntrustedEndpointOauthLiteralSecretRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_oauth_literal_secret_rejected\",\"description\":\"kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-oauth-literal\",\"model\":{\"id\":\"gpt-4\",\"provider\":\"anthropic\",\"connection\":{\"kind\":\"oauth\",\"endpoint\":\"https://attacker.example.com\",\"clientId\":\"decoy\",\"clientSecret\":\"decoy\",\"tokenUrl\":\"https://login.microsoftonline.com/t/oauth2/v2.0/token\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector68LoadConformanceLoadUntrustedEndpointPercentEncodedHostRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_percent_encoded_host_rejected\",\"description\":\"Hosts are compared without percent-decoding, so an encoded host never matches\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-percent-encoded-host\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com%2eevil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector69LoadConformanceLoadUntrustedEndpointPortMismatchRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_port_mismatch_rejected\",\"description\":\"A non-default port is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-port-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com:8443\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector70LoadConformanceLoadUntrustedEndpointQueryBeforeAtRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_query_before_at_rejected\",\"description\":\"A '?' ends the authority, so the host is evil.example.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-query-before-at\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evil.example.com?@api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector71LoadConformanceLoadUntrustedEndpointSchemeMismatchRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_scheme_mismatch_rejected\",\"description\":\"Built-in origins are https; http on the same host is a different origin\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-scheme-mismatch\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://api.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector72LoadConformanceLoadUntrustedEndpointSuffixLookalikeRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_suffix_lookalike_rejected\",\"description\":\"A trusted host as a label prefix of another domain is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-suffix-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector73LoadConformanceLoadUntrustedEndpointSuffixSpoofRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_suffix_spoof_rejected\",\"description\":\"Origin matching is exact: a host that merely starts with a trusted host is not trusted\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-spoof\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://good.example.com.attacker.example.com\"}}},\"trustedEndpoints\":[\"https://good.example.com\"]},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector74LoadConformanceLoadUntrustedEndpointTrailingDotRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_trailing_dot_rejected\",\"description\":\"Hosts are compared exactly; a trailing-dot host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-trailing-dot\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com.\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector75LoadConformanceLoadUntrustedEndpointUnicodeHomoglyphRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_unicode_homoglyph_rejected\",\"description\":\"A non-ASCII lookalike host does not match\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-unicode-homoglyph\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://\u{430}pi.openai.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector76LoadConformanceLoadUntrustedEndpointUserinfoLookalikeRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_userinfo_lookalike_rejected\",\"description\":\"A trusted host in userinfo is not the destination host\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-userinfo-lookalike\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://api.openai.com@evil.example.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector77LoadConformanceLoadUntrustedEndpointWildcardApexRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_wildcard_apex_rejected\",\"description\":\"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-apex\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"http://myres.openai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector78LoadConformanceLoadUntrustedEndpointWildcardBoundaryRejected() async throws { + guard let vectorData = "{\"name\":\"untrusted_endpoint_wildcard_boundary_rejected\",\"description\":\"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com\",\"stage\":\"load\",\"input\":{\"frontmatter\":{\"name\":\"untrusted-wildcard-boundary\",\"model\":{\"id\":\"gpt-4\",\"connection\":{\"kind\":\"foundry\",\"endpoint\":\"https://evilservices.ai.azure.com\"}}}},\"expectedError\":{\"kind\":\"untrusted_endpoint\"},\"operation\":\"load\"}".data(using: .utf8), + let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] + else { + XCTFail("failed to parse embedded vector") + return + } + _ = await VectorRunner.runVector(contract: "LoadConformance", operation: "load", vector: vector, sync: false, seam: seam()) + } + + func testVector79MemoryConformanceOperateClearByCategoryAndAll() async throws { guard let vectorData = "{\"name\":\"clear_by_category_and_all\",\"description\":\"Clear removes only the requested category, or all memories when no category is supplied.\",\"stage\":\"memory\",\"input\":{\"operation\":\"clear\",\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"summary\",\"category\":\"archival\"},{\"content\":\"another summary\",\"category\":\"archival\"},{\"content\":\"insight\",\"category\":\"insight\"}]},\"category\":\"archival\"},\"expected\":{\"removed\":2,\"store\":{\"entries\":[{\"content\":\"fact\",\"category\":\"core\"},{\"content\":\"insight\",\"category\":\"insight\"}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -524,7 +834,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector49MemoryConformanceOperateFormatCoreAndRecallResults() async throws { + func testVector80MemoryConformanceOperateFormatCoreAndRecallResults() async throws { guard let vectorData = "{\"name\":\"format_core_and_recall_results\",\"description\":\"Formatting injects only core memories into system prompts and formats recall results with category and tags.\",\"stage\":\"memory\",\"input\":{\"operation\":\"format\",\"store\":{\"entries\":[{\"content\":\"persistent fact\",\"category\":\"core\"},{\"content\":\"a summary\",\"category\":\"archival\"},{\"content\":\"run the deploy\",\"category\":\"core\",\"tags\":[\"ops\"]}]},\"query\":\"deploy\"},\"expected\":{\"system_prompt\":\"## Memory\\n- persistent fact\\n- run the deploy\\n\",\"recall_results\":\"1. [core] run the deploy\\n tags: ops\\n\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -534,7 +844,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector50MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { + func testVector81MemoryConformanceOperateRecallEmptyQueryAndLimit() async throws { guard let vectorData = "{\"name\":\"recall_empty_query_and_limit\",\"description\":\"Empty-query recall returns all memories in insertion order with score 0; positive limit truncates.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"first\",\"category\":\"insight\",\"createdAt\":\"2024-01-03T00:00:00Z\"},{\"content\":\"second\",\"category\":\"core\",\"createdAt\":\"2024-01-01T00:00:00Z\"},{\"content\":\"third\",\"category\":\"archival\",\"createdAt\":\"2024-01-02T00:00:00Z\"}]},\"query\":\"\",\"limit\":2},\"expected\":{\"results\":[{\"content\":\"first\",\"category\":\"insight\",\"score\":0,\"keyword_matches\":0},{\"content\":\"second\",\"category\":\"core\",\"score\":0,\"keyword_matches\":0}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -544,7 +854,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector51MemoryConformanceOperateRecallRanksByWeightedScore() async throws { + func testVector82MemoryConformanceOperateRecallRanksByWeightedScore() async throws { guard let vectorData = "{\"name\":\"recall_ranks_by_weighted_score\",\"description\":\"Recall ranks by weighted lexical score: content match = 2, tag match = 3, core match boost = 1.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"the sky is clear\",\"category\":\"insight\"},{\"content\":\"favorite color is blue sky\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"space\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"blue sky\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"favorite color is blue sky\",\"category\":\"insight\",\"score\":4,\"keyword_matches\":2},{\"content\":\"the sky is clear\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -554,7 +864,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector52MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { + func testVector83MemoryConformanceOperateRecallScoreTiesPreserveInsertionOrder() async throws { guard let vectorData = "{\"name\":\"recall_score_ties_preserve_insertion_order\",\"description\":\"Recall preserves insertion order for memories with identical scores.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"alpha first\",\"category\":\"insight\"},{\"content\":\"alpha second\",\"category\":\"insight\"},{\"content\":\"alpha third\",\"category\":\"insight\"}]},\"query\":\"alpha\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"alpha first\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha second\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1},{\"content\":\"alpha third\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -564,7 +874,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector53MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { + func testVector84MemoryConformanceOperateRecallTagsAndCoreBoost() async throws { guard let vectorData = "{\"name\":\"recall_tags_and_core_boost\",\"description\":\"Recall gives tag matches higher weight than content matches and boosts matching core memories.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"all about deploy\",\"category\":\"archival\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"deploy\"]},{\"content\":\"always deploy on green\",\"category\":\"core\"}]},\"query\":\"deploy\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"always deploy on green\",\"category\":\"core\",\"score\":3,\"keyword_matches\":1},{\"content\":\"all about deploy\",\"category\":\"archival\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -574,7 +884,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector54MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { + func testVector85MemoryConformanceOperateRecallTokenizationPunctuationUnicodeAndDedup() async throws { guard let vectorData = "{\"name\":\"recall_tokenization_punctuation_unicode_and_dedup\",\"description\":\"Recall trims punctuation, lowercases tokens, preserves Unicode letters, and counts duplicate query tokens once.\",\"stage\":\"memory\",\"input\":{\"operation\":\"recall\",\"store\":{\"entries\":[{\"content\":\"deploy guide\",\"category\":\"insight\"},{\"content\":\"unrelated text\",\"category\":\"insight\",\"tags\":[\"caf\u{e9}\"]},{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\"}]},\"query\":\"DEPLOY, deploy!!! caf\u{e9}\",\"limit\":0},\"expected\":{\"results\":[{\"content\":\"deploy caf\u{e9}\",\"category\":\"core\",\"score\":5,\"keyword_matches\":2},{\"content\":\"unrelated text\",\"category\":\"insight\",\"score\":3,\"keyword_matches\":1},{\"content\":\"deploy guide\",\"category\":\"insight\",\"score\":2,\"keyword_matches\":1}]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -584,7 +894,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector55MemoryConformanceOperateRememberDedupsCoreByTags() async throws { + func testVector86MemoryConformanceOperateRememberDedupsCoreByTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_by_tags\",\"description\":\"Remember replaces an existing core memory with identical tags, appends the new memory, and leaves non-core entries intact.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old fact\",\"category\":\"core\",\"tags\":[\"subject\"]},{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]}]},\"entry\":{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"summary\",\"category\":\"archival\",\"tags\":[\"subject\"]},{\"content\":\"new fact\",\"category\":\"core\",\"tags\":[\"subject\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -594,7 +904,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector56MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { + func testVector87MemoryConformanceOperateRememberDedupsCoreMissingAndEmptyTags() async throws { guard let vectorData = "{\"name\":\"remember_dedups_core_missing_and_empty_tags\",\"description\":\"Remember treats missing and empty tag lists as equivalent for core-memory deduplication.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"old core with missing tags\",\"category\":\"core\"},{\"content\":\"old core with empty tags\",\"category\":\"core\",\"tags\":[]},{\"content\":\"non-core survives\",\"category\":\"insight\"}]},\"entry\":{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]},\"max_entries\":0},\"expected\":{\"store\":{\"entries\":[{\"content\":\"non-core survives\",\"category\":\"insight\"},{\"content\":\"new core with empty tags\",\"category\":\"core\",\"tags\":[]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -604,7 +914,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector57MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { + func testVector88MemoryConformanceOperateRememberEvictsArchivalFirst() async throws { guard let vectorData = "{\"name\":\"remember_evicts_archival_first\",\"description\":\"Remember enforces the cap by evicting the oldest archival memory before older core/insight entries.\",\"stage\":\"memory\",\"input\":{\"operation\":\"remember\",\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"archival b\",\"category\":\"archival\"}]},\"entry\":{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]},\"max_entries\":2},\"expected\":{\"store\":{\"entries\":[{\"content\":\"core a\",\"category\":\"core\",\"tags\":[\"a\"]},{\"content\":\"core c\",\"category\":\"core\",\"tags\":[\"c\"]}]}},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -614,7 +924,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector58MemoryConformanceOperateSnapshotPortRoundTrip() async throws { + func testVector89MemoryConformanceOperateSnapshotPortRoundTrip() async throws { guard let vectorData = "{\"name\":\"snapshot_port_round_trip\",\"description\":\"A host memory port persists and reloads the whole MemoryStore snapshot; recall runs against the reloaded snapshot.\",\"stage\":\"memory\",\"input\":{\"operation\":\"snapshot\",\"store\":{\"entries\":[]},\"entry\":{\"content\":\"remember this\",\"category\":\"core\"},\"max_entries\":200,\"query\":\"remember\"},\"expected\":{\"store\":{\"entries\":[{\"content\":\"remember this\",\"category\":\"core\"}]},\"recalled\":[\"remember this\"]},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -624,7 +934,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector59MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { + func testVector90MemoryConformanceOperateUpdateIndexOutOfRangeError() async throws { guard let vectorData = "{\"name\":\"update_index_out_of_range_error\",\"description\":\"Updating a missing memory index signals a stable out-of-range error kind without relying on runtime-specific message text.\",\"stage\":\"memory\",\"input\":{\"operation\":\"update\",\"store\":{\"entries\":[{\"content\":\"only\",\"category\":\"core\"}]},\"index\":3,\"entry\":{\"content\":\"replacement\",\"category\":\"insight\"}},\"expectedError\":{\"kind\":\"index_out_of_range\"},\"operation\":\"operate\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -634,7 +944,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "MemoryConformance", operation: "operate", vector: vector, sync: false, seam: seam()) } - func testVector60TurnConformanceReplayMaxIterations() async throws { + func testVector91TurnConformanceReplayMaxIterations() async throws { guard let vectorData = "{\"name\":\"max_iterations\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"maxIterations\":1},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:error:1:max_iterations\",\"turn:turn_end:1:error\",\"session:session_end:session-1:turn-1:error\",\"summary:session-1:error:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -644,7 +954,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector61TurnConformanceReplayNoTool() async throws { + func testVector92TurnConformanceReplayNoTool() async throws { guard let vectorData = "{\"name\":\"no_tool\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\",\"inputs\":{\"name\":\"Ada\"},\"maxIterations\":3},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:1:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=1\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -654,7 +964,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector62TurnConformanceReplayPermissionDenied() async throws { + func testVector93TurnConformanceReplayPermissionDenied() async throws { guard let vectorData = "{\"name\":\"permission_denied\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:false\",\"turn:tool_result:0:add:false:permission_denied\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -664,7 +974,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector63TurnConformanceReplayToolFailure() async throws { + func testVector94TurnConformanceReplayToolFailure() async throws { guard let vectorData = "{\"name\":\"tool_failure\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:fail\",\"turn:tool_execution_complete:0:fail:false:exception\",\"turn:tool_result:0:fail:false:exception\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -674,7 +984,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector64TurnConformanceReplayToolSuccess() async throws { + func testVector95TurnConformanceReplayToolSuccess() async throws { guard let vectorData = "{\"name\":\"tool_success\",\"stage\":\"replay\",\"input\":{\"clock\":\"2026-06-28T00:00:00Z\",\"sessionId\":\"session-1\",\"turnId\":\"turn-1\"},\"expected\":[\"session:session_start:session-1:turn-1\",\"turn:turn_start:0\",\"turn:llm_start:0\",\"turn:llm_complete:0\",\"session:checkpoint_created:session-1:turn-1\",\"turn:permission_requested:0:exec-1-permission\",\"turn:permission_completed:0:true\",\"turn:tool_execution_start:0:add\",\"turn:tool_execution_complete:0:add:true\",\"turn:tool_result:0:add:true\",\"turn:messages_updated:0\",\"turn:llm_start:1\",\"turn:llm_complete:1\",\"session:checkpoint_created:session-1:turn-1\",\"turn:turn_end:2:success\",\"session:session_end:session-1:turn-1:success\",\"summary:session-1:success:turns=1:checkpoints=2\"],\"operation\":\"replay\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -684,7 +994,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "replay", vector: vector, sync: false, seam: seam()) } - func testVector65TurnConformanceRunAssistantToolCallsMetadata() async throws { + func testVector96TurnConformanceRunAssistantToolCallsMetadata() async throws { guard let vectorData = "{\"name\":\"assistant_tool_calls_metadata\",\"description\":\"When the LLM returns tool calls, the assistant message appended to the conversation must have role='assistant', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in two cities.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-meta-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_meta_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_meta_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_meta_paris\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_meta_london\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-meta-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is warmer and sunnier than London today.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is warmer and sunnier than London today.\",\"iterations\":2,\"assistant_tool_calls_message\":{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_meta_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_meta_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -694,7 +1004,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector66TurnConformanceRunAsyncToolFunction() async throws { + func testVector97TurnConformanceRunAsyncToolFunction() async throws { guard let vectorData = "{\"name\":\"async_tool_function\",\"description\":\"Agent loop correctly handles async tool functions\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are helpful.\"},{\"role\":\"user\",\"content\":\"Look up the data\"}],\"tools\":[{\"name\":\"lookup\",\"kind\":\"function\",\"description\":\"Look up data\",\"parameters\":[{\"name\":\"query\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"lookup\":\"returns lookup result\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-async-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_async1\",\"type\":\"function\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"query\\\":\\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_async1\",\"name\":\"lookup\",\"arguments\":{\"query\":\"test\"}}],\"tool_results\":[{\"tool_call_id\":\"call_async1\",\"result\":\"found: test data\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-async-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I found: test data\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I found: test data\",\"iterations\":2,\"total_messages\":6},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -704,7 +1014,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector67TurnConformanceRunBindingsInjected() async throws { + func testVector98TurnConformanceRunBindingsInjected() async throws { guard let vectorData = "{\"name\":\"bindings_injected\",\"description\":\"Tool has a binding {unit: {input: 'preferred_unit'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"parent_inputs\":{\"preferred_unit\":\"celsius\"},\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":false}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"tool_functions\":{\"get_weather\":\"returns weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-binding-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_bind_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_bind_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"expected_execution_args\":{\"get_weather\":{\"city\":\"Paris\",\"unit\":\"celsius\"}},\"tool_results\":[{\"tool_call_id\":\"call_bind_001\",\"result\":\"22\u{b0}C sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-binding-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22\u{b0}C and sunny.\",\"iterations\":2,\"total_messages\":6,\"notes\":\"The binding {unit: {input: 'preferred_unit'}} resolves 'preferred_unit' from parent_inputs to 'celsius' and injects it into the tool call arguments before execution, overriding any LLM-provided value.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -714,7 +1024,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector68TurnConformanceRunCancellationBeforeLlm() async throws { + func testVector99TurnConformanceRunCancellationBeforeLlm() async throws { guard let vectorData = "{\"name\":\"cancellation_before_llm\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"cancel\":{\"cancelled_at\":\"before_iteration\"},\"on_event\":true},\"sequence\":[],\"expected\":{\"error\":\"CancelledError\",\"iterations\":0,\"total_messages\":2,\"events\":[{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before first iteration\"}}],\"notes\":\"No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -724,7 +1034,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector69TurnConformanceRunCancellationBetweenIterations() async throws { + func testVector100TurnConformanceRunCancellationBetweenIterations() async throws { guard let vectorData = "{\"name\":\"cancellation_between_iterations\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"before_iteration_2\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-iter-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_iter001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_iter001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_iter001\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"total_messages\":5,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested before iteration 2\"}}],\"notes\":\"Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -734,7 +1044,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector70TurnConformanceRunCancellationBetweenTools() async throws { + func testVector101TurnConformanceRunCancellationBetweenTools() async throws { guard let vectorData = "{\"name\":\"cancellation_between_tools\",\"description\":\"\u{a7}13.2 Cancellation \u{2014} LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Compare weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"cancel\":{\"cancelled_at\":\"after_tool_0\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-cancel-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_cancel_paris\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_cancel_london\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_cancel_paris\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_cancel_london\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_cancel_paris\",\"result\":\"72\u{b0}F sunny\"}]}],\"expected\":{\"error\":\"CancelledError\",\"iterations\":1,\"tools_executed\":1,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"cancelled\",\"data\":{\"reason\":\"Cancellation requested after tool execution\"}}],\"notes\":\"Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -744,7 +1054,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector71TurnConformanceRunContextNoTrimWhenFits() async throws { + func testVector102TurnConformanceRunContextNoTrimWhenFits() async throws { guard let vectorData = "{\"name\":\"context_no_trim_when_fits\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages fit within the context budget. No trimming occurs and all messages are preserved.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"context_budget\":10000},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-nofit-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"trimmed_messages\":null,\"notes\":\"When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -754,7 +1064,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector72TurnConformanceRunContextPreservesSystemMessages() async throws { + func testVector103TurnConformanceRunContextPreservesSystemMessages() async throws { guard let vectorData = "{\"name\":\"context_preserves_system_messages\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"user\",\"content\":\"Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail.\"},{\"role\":\"assistant\",\"content\":\"Ancient Egypt's history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun.\"},{\"role\":\"user\",\"content\":\"Now tell me about Greek philosophy covering all major philosophers and their contributions.\"},{\"role\":\"assistant\",\"content\":\"Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":300},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-sys-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I would need to know your city to provide the weather.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"I would need to know your city to provide the weather.\",\"iterations\":1,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"Always respond in a formal tone.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers.\"},{\"role\":\"user\",\"content\":\"What is the weather today?\"}],\"notes\":\"ALL system messages are preserved during context trimming \u{2014} they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -764,7 +1074,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector73TurnConformanceRunContextTrimBasic() async throws { + func testVector104TurnConformanceRunContextTrimBasic() async throws { guard let vectorData = "{\"name\":\"context_trim_basic\",\"description\":\"\u{a7}13.3 Context Window \u{2014} Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me about the history of Rome. I want a very detailed answer covering all major periods.\"},{\"role\":\"assistant\",\"content\":\"Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire.\"},{\"role\":\"user\",\"content\":\"Now tell me about the fall of Rome. Again, very detailed with all the key events and dates.\"},{\"role\":\"assistant\",\"content\":\"The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453.\"},{\"role\":\"user\",\"content\":\"Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events.\"},{\"role\":\"assistant\",\"content\":\"After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095.\"},{\"role\":\"user\",\"content\":\"Now tell me about the Renaissance period.\"},{\"role\":\"assistant\",\"content\":\"The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\"},\"context_budget\":500},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ctx_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_ctx_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_ctx_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-ctx-trim-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"trimmed_messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"system\",\"content\":\"[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period.\"},{\"role\":\"user\",\"content\":\"Finally, what is the weather in Paris today?\"}],\"summary_contains\":\"User asked\",\"notes\":\"Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -774,7 +1084,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector74TurnConformanceRunEmptyToolResult() async throws { + func testVector105TurnConformanceRunEmptyToolResult() async throws { guard let vectorData = "{\"name\":\"empty_tool_result\",\"description\":\"Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content \u{2014} it must not be skipped or filtered out.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Clear the cache.\"}],\"tools\":[{\"name\":\"clear_cache\",\"kind\":\"function\",\"description\":\"Clear the application cache, returns empty on success\",\"parameters\":[]}],\"tool_functions\":{\"clear_cache\":\"returns empty string on success\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-empty-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_empty_001\",\"type\":\"function\",\"function\":{\"name\":\"clear_cache\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_empty_001\",\"name\":\"clear_cache\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_empty_001\",\"result\":\"\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-empty-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The cache has been cleared successfully.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The cache has been cleared successfully.\",\"iterations\":2,\"total_messages\":6,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"\"}],\"metadata\":{\"tool_call_id\":\"call_empty_001\"}},\"notes\":\"Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -784,7 +1094,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector75TurnConformanceRunEventsBasicToolLoop() async throws { + func testVector106TurnConformanceRunEventsBasicToolLoop() async throws { guard let vectorData = "{\"name\":\"events_basic_tool_loop\",\"description\":\"\u{a7}13.1 Events \u{2014} A 2-turn tool-call loop emits the full event lifecycle: status \u{2192} tool_call_start \u{2192} tool_result \u{2192} messages_updated \u{2192} done.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-basic-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"done\",\"data\":{\"response\":\"The weather in Paris is 72\u{b0}F and sunny.\"}}],\"notes\":\"Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -794,7 +1104,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector76TurnConformanceRunEventsErrorLogged() async throws { + func testVector107TurnConformanceRunEventsErrorLogged() async throws { guard let vectorData = "{\"name\":\"events_error_logged\",\"description\":\"\u{a7}13.1 Events \u{97} A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"raises RuntimeError\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-err-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_evt_err001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_evt_err001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_evt_err001\",\"result\":\"Error calling 'get_weather': RuntimeError: Weather service unavailable\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-evt-err-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sorry, the weather service is currently unavailable.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Sorry, the weather service is currently unavailable.\",\"iterations\":2,\"total_messages\":6,\"events\":[{\"type\":\"tool_call_start\"},{\"type\":\"tool_result\"},{\"type\":\"messages_updated\"},{\"type\":\"done\"}],\"notes\":\"Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues \u{97} this is a success path. Events verify type sequence only (payloads vary by runtime).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -804,7 +1114,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector77TurnConformanceRunEventsNoTools() async throws { + func testVector108TurnConformanceRunEventsNoTools() async throws { guard let vectorData = "{\"name\":\"events_no_tools\",\"description\":\"\u{a7}13.1 Events \u{2014} Single-turn completion with no tool calls. Only the done event is emitted.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-evt-notool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3,\"events\":[{\"type\":\"done\",\"data\":{\"response\":\"2 + 2 equals 4.\"}}],\"notes\":\"When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -814,7 +1124,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector78TurnConformanceRunGuardrailAllPass() async throws { + func testVector109TurnConformanceRunGuardrailAllPass() async throws { guard let vectorData = "{\"name\":\"guardrail_all_pass\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"allow\"},\"tool\":{\"deny_tools\":[]}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_pass001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_pass001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_pass001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-pass-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"denied_tools\":[],\"notes\":\"All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -824,7 +1134,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector79TurnConformanceRunGuardrailInputDeny() async throws { + func testVector110TurnConformanceRunGuardrailInputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_input_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"My SSN is 123-45-6789, can you look that up?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"deny\",\"reason\":\"Contains PII\"}}},\"sequence\":[],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Contains PII\",\"iterations\":0,\"total_messages\":2,\"notes\":\"Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -834,7 +1144,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector80TurnConformanceRunGuardrailOutputDeny() async throws { + func testVector111TurnConformanceRunGuardrailOutputDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_output_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Tell me something.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"},\"guardrails\":{\"input\":{\"action\":\"allow\"},\"output\":{\"action\":\"deny\",\"reason\":\"Response contains harmful content\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-out-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here is some problematic content that the guardrail catches.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"error\":\"GuardrailError\",\"error_reason\":\"Response contains harmful content\",\"iterations\":1,\"notes\":\"The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -844,7 +1154,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector81TurnConformanceRunGuardrailToolDeny() async throws { + func testVector112TurnConformanceRunGuardrailToolDeny() async throws { guard let vectorData = "{\"name\":\"guardrail_tool_deny\",\"description\":\"\u{a7}13.4 Guardrails \u{2014} LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get the weather and also run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation that should be guarded\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"dangerous_tool\":\"returns sensitive data\"},\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Tool not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_guard_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_guard_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_guard_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_guard_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}}],\"tool_results\":[{\"tool_call_id\":\"call_guard_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_guard_danger\",\"result\":\"Tool denied by guardrail: Tool not authorized\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-guard-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny. I was unable to run the dangerous operation as it was not authorized.\",\"iterations\":2,\"total_messages\":7,\"denied_tools\":[\"dangerous_tool\"],\"notes\":\"The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied \u{2014} its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -854,7 +1164,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector82TurnConformanceRunMaxIterationsExceeded() async throws { + func testVector113TurnConformanceRunMaxIterationsExceeded() async throws { guard let vectorData = "{\"name\":\"max_iterations_exceeded\",\"description\":\"LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Keep checking the weather forever.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-loop-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_01\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_01\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_01\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-loop-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_02\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_02\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_02\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-loop-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_03\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_03\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_03\",\"result\":\"80\u{b0}F humid\"}]},{\"turn\":4,\"llm_response\":{\"id\":\"chatcmpl-loop-004\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_04\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Sydney\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_04\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Sydney\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_04\",\"result\":\"65\u{b0}F rainy\"}]},{\"turn\":5,\"llm_response\":{\"id\":\"chatcmpl-loop-005\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_05\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Berlin\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_05\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Berlin\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_05\",\"result\":\"55\u{b0}F windy\"}]},{\"turn\":6,\"llm_response\":{\"id\":\"chatcmpl-loop-006\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_06\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Cairo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_06\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Cairo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_06\",\"result\":\"95\u{b0}F dry\"}]},{\"turn\":7,\"llm_response\":{\"id\":\"chatcmpl-loop-007\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_07\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Mumbai\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_07\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Mumbai\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_07\",\"result\":\"88\u{b0}F humid\"}]},{\"turn\":8,\"llm_response\":{\"id\":\"chatcmpl-loop-008\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_08\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Moscow\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_08\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Moscow\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_08\",\"result\":\"30\u{b0}F snowy\"}]},{\"turn\":9,\"llm_response\":{\"id\":\"chatcmpl-loop-009\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_09\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Rio\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_09\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rio\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_09\",\"result\":\"85\u{b0}F tropical\"}]},{\"turn\":10,\"llm_response\":{\"id\":\"chatcmpl-loop-010\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_10\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Seoul\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_10\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Seoul\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_10\",\"result\":\"68\u{b0}F clear\"}]},{\"turn\":11,\"llm_response\":{\"id\":\"chatcmpl-loop-011\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_loop_11\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Toronto\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_loop_11\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Toronto\"}}],\"tool_results\":[{\"tool_call_id\":\"call_loop_11\",\"result\":\"40\u{b0}F foggy\"}]}],\"expected\":{\"error\":\"Agent loop exceeded 10 iterations\",\"iterations\":11},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -864,7 +1174,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector83TurnConformanceRunMultiTurnToolCalls() async throws { + func testVector114TurnConformanceRunMultiTurnToolCalls() async throws { guard let vectorData = "{\"name\":\"multi_turn_tool_calls\",\"description\":\"Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather and conversion tools.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris in Celsius?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city (returns Fahrenheit)\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"convert_temperature\",\"kind\":\"function\",\"description\":\"Convert a temperature between Fahrenheit and Celsius\",\"parameters\":[{\"name\":\"value\",\"kind\":\"float\",\"required\":true},{\"name\":\"from_unit\",\"kind\":\"string\",\"required\":true},{\"name\":\"to_unit\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string in Fahrenheit\",\"convert_temperature\":\"returns converted temperature string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-chain-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_weather_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_weather_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_weather_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-chain-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_convert_001\",\"type\":\"function\",\"function\":{\"name\":\"convert_temperature\",\"arguments\":\"{\\\"value\\\": 72.0, \\\"from_unit\\\": \\\"fahrenheit\\\", \\\"to_unit\\\": \\\"celsius\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_convert_001\",\"name\":\"convert_temperature\",\"arguments\":{\"value\":72,\"from_unit\":\"fahrenheit\",\"to_unit\":\"celsius\"}}],\"tool_results\":[{\"tool_call_id\":\"call_convert_001\",\"result\":\"22.2\u{b0}C\"}]},{\"turn\":3,\"llm_response\":{\"id\":\"chatcmpl-chain-003\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 22.2\u{b0}C and sunny.\",\"iterations\":3,\"total_messages\":8},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -874,7 +1184,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector84TurnConformanceRunMultipleToolCallsSingleTurn() async throws { + func testVector115TurnConformanceRunMultipleToolCallsSingleTurn() async throws { guard let vectorData = "{\"name\":\"multiple_tool_calls_single_turn\",\"description\":\"Turn 1: LLM returns two parallel tool calls \u{2014} get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-multi-call-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_paris_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_london_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"London\"}}],\"tool_results\":[{\"tool_call_id\":\"call_paris_001\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_london_001\",\"result\":\"59\u{b0}F cloudy\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-multi-call-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\",\"iterations\":2,\"total_messages\":7,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"Compare the weather in Paris and London.\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_paris_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_london_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"London\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_paris_001\"}},{\"role\":\"tool\",\"content\":\"59\u{b0}F cloudy\",\"metadata\":{\"tool_call_id\":\"call_london_001\"}},{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny, while London is 59\u{b0}F and cloudy.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -884,7 +1194,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector85TurnConformanceRunNoToolCalls() async throws { + func testVector116TurnConformanceRunNoToolCalls() async throws { guard let vectorData = "{\"name\":\"no_tool_calls\",\"description\":\"LLM returns content immediately with no tool calls \u{2014} the agent loop completes in a single iteration.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is 2 + 2?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-no-tools-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"2 + 2 equals 4.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"2 + 2 equals 4.\",\"iterations\":1,\"total_messages\":3},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -894,7 +1204,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector86TurnConformanceRunParallelToolsBasic() async throws { + func testVector117TurnConformanceRunParallelToolsBasic() async throws { guard let vectorData = "{\"name\":\"parallel_tools_basic\",\"description\":\"\u{a7}13.6 Parallel Tools \u{2014} LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and news tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, the current time in Tokyo, and the latest news.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_news\",\"kind\":\"function\",\"description\":\"Get the latest news headlines\",\"parameters\":[]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"get_news\":\"returns news headlines\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_par_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_par_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}},{\"id\":\"call_par_news\",\"type\":\"function\",\"function\":{\"name\":\"get_news\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_par_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_par_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}},{\"id\":\"call_par_news\",\"name\":\"get_news\",\"arguments\":{}}],\"tool_results\":[{\"tool_call_id\":\"call_par_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_par_time\",\"result\":\"3:45 PM JST\"},{\"tool_call_id\":\"call_par_news\",\"result\":\"Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice.\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Here's your update: Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_time\",\"get_news\"],\"notes\":\"With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -904,7 +1214,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector87TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { + func testVector118TurnConformanceRunParallelToolsWithGuardrailDeny() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_guardrail_deny\",\"description\":\"\u{a7}13.6 Parallel Tools + \u{a7}13.4 Guardrails \u{2014} 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Get weather, time, and run the dangerous operation.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"dangerous_tool\",\"kind\":\"function\",\"description\":\"A dangerous operation\",\"parameters\":[{\"name\":\"target\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_time\":\"returns current time string\",\"dangerous_tool\":\"returns sensitive data\"},\"parallel_tool_calls\":true,\"guardrails\":{\"tool\":{\"deny_tools\":[\"dangerous_tool\"],\"reason\":\"Not authorized\"}}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-par-guard-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pg_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pg_danger\",\"type\":\"function\",\"function\":{\"name\":\"dangerous_tool\",\"arguments\":\"{\\\"target\\\": \\\"secret_db\\\"}\"}},{\"id\":\"call_pg_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pg_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pg_danger\",\"name\":\"dangerous_tool\",\"arguments\":{\"target\":\"secret_db\"}},{\"id\":\"call_pg_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pg_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pg_danger\",\"result\":\"Tool denied by guardrail: Not authorized\"},{\"tool_call_id\":\"call_pg_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-par-guard-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.\",\"iterations\":2,\"total_messages\":8,\"denied_tools\":[\"dangerous_tool\"],\"tool_execution_order\":[\"get_weather\",\"get_time\"],\"notes\":\"The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -914,7 +1224,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector88TurnConformanceRunParallelToolsWithToolError() async throws { + func testVector119TurnConformanceRunParallelToolsWithToolError() async throws { guard let vectorData = "{\"name\":\"parallel_tools_with_tool_error\",\"description\":\"\u{a7}13.6 Parallel Tools + tool error \u{2014} 3 parallel tool calls, one tool returns an error result. All result messages are appended in the original tool_calls order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to weather, time, and service-status tools.\"},{\"role\":\"user\",\"content\":\"Give me the weather in Paris, service status, and the time in Tokyo.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]},{\"name\":\"get_service_status\",\"kind\":\"function\",\"description\":\"Get service status\",\"parameters\":[]},{\"name\":\"get_time\",\"kind\":\"function\",\"description\":\"Get the current time in a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string\",\"get_service_status\":\"raises RuntimeError\",\"get_time\":\"returns current time string\"},\"parallel_tool_calls\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_pe_weather\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"id\":\"call_pe_status\",\"type\":\"function\",\"function\":{\"name\":\"get_service_status\",\"arguments\":\"{}\"}},{\"id\":\"call_pe_time\",\"type\":\"function\",\"function\":{\"name\":\"get_time\",\"arguments\":\"{\\\"city\\\": \\\"Tokyo\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_pe_weather\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}},{\"id\":\"call_pe_status\",\"name\":\"get_service_status\",\"arguments\":{}},{\"id\":\"call_pe_time\",\"name\":\"get_time\",\"arguments\":{\"city\":\"Tokyo\"}}],\"tool_results\":[{\"tool_call_id\":\"call_pe_weather\",\"result\":\"72\u{b0}F sunny\"},{\"tool_call_id\":\"call_pe_status\",\"result\":\"Error calling 'get_service_status': RuntimeError: Service unavailable\"},{\"tool_call_id\":\"call_pe_time\",\"result\":\"3:45 PM JST\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-parallel-error-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Paris is 72\u{b0}F and sunny. The service status is unavailable right now. The time in Tokyo is 3:45 PM JST.\",\"iterations\":2,\"total_messages\":8,\"tool_execution_order\":[\"get_weather\",\"get_service_status\",\"get_time\"],\"notes\":\"A tool error is represented as that tool's result content, not by reordering or dropping the slot. Runtimes may execute effects concurrently or sequentially, but provider-visible tool result messages remain aligned to the original tool_calls array.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -924,7 +1234,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector89TurnConformanceRunSingleToolCall() async throws { + func testVector120TurnConformanceRunSingleToolCall() async throws { guard let vectorData = "{\"name\":\"single_tool_call\",\"description\":\"Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-single-tool-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_abc123\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_abc123\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-single-tool-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The weather in Paris is 72\u{b0}F and sunny.\",\"iterations\":2,\"total_messages\":6,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_abc123\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_abc123\"}},{\"role\":\"assistant\",\"content\":\"The weather in Paris is 72\u{b0}F and sunny.\"}]},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -934,7 +1244,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector90TurnConformanceRunSteeringInjectMessage() async throws { + func testVector121TurnConformanceRunSteeringInjectMessage() async throws { guard let vectorData = "{\"name\":\"steering_inject_message\",\"description\":\"\u{a7}13.5 Steering \u{2014} After turn 1 (tool call + result), a steering message is injected before turn 2's LLM call. The LLM sees the injected message in context.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Actually, focus on the temperature in Celsius.\"}]},\"on_event\":true},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_w001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_w001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\",\"iterations\":2,\"total_messages\":7,\"events\":[{\"type\":\"status\",\"data\":{\"message\":\"Starting agent loop\"}},{\"type\":\"tool_call_start\",\"data\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}},{\"type\":\"tool_result\",\"data\":{\"name\":\"get_weather\",\"result\":\"72\u{b0}F sunny\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":5}},{\"type\":\"status\",\"data\":{\"message\":\"Injecting steering message\"}},{\"type\":\"messages_updated\",\"data\":{\"message_count\":6}},{\"type\":\"done\",\"data\":{\"response\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}}],\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful weather assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_w001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny\",\"metadata\":{\"tool_call_id\":\"call_steer_w001\"}},{\"role\":\"user\",\"content\":\"Actually, focus on the temperature in Celsius.\"},{\"role\":\"assistant\",\"content\":\"The temperature in Paris is approximately 22\u{b0}C and it is sunny.\"}],\"notes\":\"The steering message is injected as a user message before iteration 2's LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call.\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -944,7 +1254,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector91TurnConformanceRunSteeringMultipleMessages() async throws { + func testVector122TurnConformanceRunSteeringMultipleMessages() async throws { guard let vectorData = "{\"name\":\"steering_multiple_messages\",\"description\":\"\u{a7}13.5 Steering \u{2014} Two steering messages are queued and both injected before iteration 2. Tests atomic drain \u{2014} both appear in order.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns weather string for the requested city\"},\"steering\":{\"messages\":[{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Please respond in French.\"},{\"inject_before_iteration\":2,\"role\":\"user\",\"text\":\"Also include the humidity percentage.\"}]}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_steer_m001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_steer_m001\",\"result\":\"72\u{b0}F sunny, 45% humidity\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-steer-multi-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\",\"iterations\":2,\"total_messages\":8,\"message_sequence\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"\",\"metadata\":{\"tool_calls\":[{\"id\":\"call_steer_m001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]}},{\"role\":\"tool\",\"content\":\"72\u{b0}F sunny, 45% humidity\",\"metadata\":{\"tool_call_id\":\"call_steer_m001\"}},{\"role\":\"user\",\"content\":\"Please respond in French.\"},{\"role\":\"user\",\"content\":\"Also include the humidity percentage.\"},{\"role\":\"assistant\",\"content\":\"Le temps \u{e0} Paris est de 22\u{b0}C et ensoleill\u{e9} avec une humidit\u{e9} de 45%.\"}],\"notes\":\"Both steering messages are drained atomically and inserted in order before iteration 2's LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles).\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -954,7 +1264,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector92TurnConformanceRunToolNotRegisteredError() async throws { + func testVector123TurnConformanceRunToolNotRegisteredError() async throws { guard let vectorData = "{\"name\":\"tool_not_registered_error\",\"description\":\"LLM calls a tool named 'unknown_tool' that has no registered handler in tool_functions. The agent loop must raise a ValueError.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"Do something with an unknown tool.\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-unknown-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_unknown_001\",\"type\":\"function\",\"function\":{\"name\":\"unknown_tool\",\"arguments\":\"{\\\"query\\\": \\\"test\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_unknown_001\",\"name\":\"unknown_tool\",\"arguments\":{\"query\":\"test\"}}]}],\"expected\":{\"error\":\"Tool not registered: unknown_tool\",\"error_type\":\"ValueError\"},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -964,7 +1274,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector93TurnConformanceRunToolResultMessageFormat() async throws { + func testVector124TurnConformanceRunToolResultMessageFormat() async throws { guard let vectorData = "{\"name\":\"tool_result_message_format\",\"description\":\"After executing a tool, the tool result message appended to the conversation must have role='tool', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.\",\"stage\":\"agent\",\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get the current weather for a city\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"tool_functions\":{\"get_weather\":\"returns a weather string\"}},\"sequence\":[{\"turn\":1,\"llm_response\":{\"id\":\"chatcmpl-format-001\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_fmt_001\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]},\"expected_tool_calls\":[{\"id\":\"call_fmt_001\",\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}],\"tool_results\":[{\"tool_call_id\":\"call_fmt_001\",\"result\":\"72\u{b0}F sunny\"}]},{\"turn\":2,\"llm_response\":{\"id\":\"chatcmpl-format-002\",\"object\":\"chat.completion\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"It is 72\u{b0}F and sunny.\",\"tool_calls\":null},\"finish_reason\":\"stop\"}]},\"expected_tool_calls\":null}],\"expected\":{\"result\":\"It is 72\u{b0}F and sunny.\",\"iterations\":2,\"tool_result_message\":{\"role\":\"tool\",\"content\":[{\"type\":\"text\",\"text\":\"72\u{b0}F sunny\"}],\"metadata\":{\"tool_call_id\":\"call_fmt_001\"}}},\"operation\":\"run\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -974,7 +1284,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "run", vector: vector, sync: false, seam: seam()) } - func testVector94TurnConformanceRunTurnCancelBeforeContext() async throws { + func testVector125TurnConformanceRunTurnCancelBeforeContext() async throws { guard let vectorData = "{\"name\":\"cancel_before_context\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Do not invoke the model\"}],\"model\":[],\"cancelBeforeRun\":true},\"expected\":{\"status\":\"cancelled\",\"iterations\":0,\"snapshots\":0,\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"turn_cancelled\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -984,7 +1294,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector95TurnConformanceRunTurnDelegatedProviderState() async throws { + func testVector126TurnConformanceRunTurnDelegatedProviderState() async throws { guard let vectorData = "{\"name\":\"delegated_provider_state\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Continue a provider-managed response\"}],\"model\":[{\"tools\":[{\"id\":\"call-state\",\"name\":\"echo\",\"arguments\":{\"value\":\"state\"}}],\"nextPortability\":\"delegated\",\"delegatedState\":[{\"provider\":\"openai\",\"kind\":\"previous_response\",\"id\":\"resp_123\"}]},{\"output\":\"continued\"}],\"toolOutputs\":{\"call-state\":\"state\"}},\"expected\":{\"status\":\"success\",\"output\":\"continued\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"snapshotPortability\":[\"portable\",\"delegated\"],\"toolResults\":1,\"toolResultOrder\":[\"call-state\"],\"commitPortability\":\"delegated\",\"delegatedState\":1},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -994,7 +1304,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector96TurnConformanceRunTurnFinalOutput() async throws { + func testVector127TurnConformanceRunTurnFinalOutput() async throws { guard let vectorData = "{\"name\":\"final_output\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"model\":[{\"output\":\"Hello back\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Hello back\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[1],\"toolResults\":0,\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1004,7 +1314,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector97TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { + func testVector128TurnConformanceRunTurnMemoryContextIsModelVisible() async throws { guard let vectorData = "{\"name\":\"memory_context_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"What should I remember?\"}],\"memory\":{\"store\":{\"entries\":[{\"content\":\"User prefers concise answers\",\"category\":\"core\"},{\"content\":\"Historical deploy summary\",\"category\":\"archival\",\"tags\":[\"deploy\"]}]}},\"model\":[{\"output\":\"Use the memory context\"}]},\"expected\":{\"status\":\"success\",\"output\":\"Use the memory context\",\"iterations\":1,\"snapshots\":1,\"snapshotStablePrefixes\":[2],\"toolResults\":0,\"providerMessages\":[{\"role\":\"system\",\"content\":\"## Memory\\n- User prefers concise answers\\n\"},{\"role\":\"user\",\"content\":\"What should I remember?\"}]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1014,7 +1324,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector98TurnConformanceRunTurnOrderedToolRound() async throws { + func testVector129TurnConformanceRunTurnOrderedToolRound() async throws { guard let vectorData = "{\"name\":\"ordered_tool_round\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Add two values\"}],\"model\":[{\"assistant\":\"I will use the tools.\",\"tools\":[{\"id\":\"call-a\",\"name\":\"echo\",\"arguments\":{\"value\":\"A\"}},{\"id\":\"call-b\",\"name\":\"echo\",\"arguments\":{\"value\":\"B\"}}]},{\"output\":\"A then B\"}],\"toolOutputs\":{\"call-a\":\"A\",\"call-b\":\"B\"}},\"expected\":{\"status\":\"success\",\"output\":\"A then B\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":2,\"toolResultOrder\":[\"call-a\",\"call-b\"],\"eventKinds\":[\"turn_started\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"permission_requested\",\"permission_resolved\",\"tool_execution_started\",\"tool_execution_completed\",\"checkpoint_created\",\"tool_result_committed\",\"tool_result_committed\",\"conversation_updated\",\"checkpoint_created\",\"context_prepared\",\"model_invocation_started\",\"model_invocation_completed\",\"checkpoint_created\",\"turn_committed\",\"post_commit_started\",\"post_commit_completed\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1024,7 +1334,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector99TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { + func testVector130TurnConformanceRunTurnPermissionDenialIsModelVisible() async throws { guard let vectorData = "{\"name\":\"permission_denial_is_model_visible\",\"stage\":\"turn\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":\"Read the protected resource\"}],\"model\":[{\"tools\":[{\"id\":\"call-denied\",\"name\":\"protected\",\"arguments\":{}}]},{\"output\":\"Permission was denied\"}],\"denyTools\":[\"protected\"]},\"expected\":{\"status\":\"success\",\"output\":\"Permission was denied\",\"iterations\":2,\"snapshots\":2,\"snapshotStablePrefixes\":[1,1],\"toolResults\":1,\"toolResultOrder\":[\"call-denied\"]},\"operation\":\"runTurn\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1034,7 +1344,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "TurnConformance", operation: "runTurn", vector: vector, sync: false, seam: seam()) } - func testVector100WireConformanceToRequestAnthropicImageFormat() async throws { + func testVector131WireConformanceToRequestAnthropicImageFormat() async throws { guard let vectorData = "{\"name\":\"anthropic_image_format\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: ImagePart uses base64 source block with media_type, not image_url format.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"base64data\",\"mediaType\":\"image/png\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/png\",\"data\":\"base64data\"}}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1044,7 +1354,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector101WireConformanceToRequestAnthropicMaxTokensRequired() async throws { + func testVector132WireConformanceToRequestAnthropicMaxTokensRequired() async throws { guard let vectorData = "{\"name\":\"anthropic_max_tokens_required\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1054,7 +1364,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector102WireConformanceToRequestAnthropicOptions() async throws { + func testVector133WireConformanceToRequestAnthropicOptions() async throws { guard let vectorData = "{\"name\":\"anthropic_options\",\"description\":\"Anthropic: option names map correctly (topK, stopSequences)\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hi\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"topP\":0.9,\"topK\":40,\"maxOutputTokens\":2000,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hi\"}]}],\"temperature\":0.5,\"top_p\":0.9,\"top_k\":40,\"max_tokens\":2000,\"stop_sequences\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1064,7 +1374,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector103WireConformanceToRequestAnthropicSystemSeparate() async throws { + func testVector134WireConformanceToRequestAnthropicSystemSeparate() async throws { guard let vectorData = "{\"name\":\"anthropic_system_separate\",\"description\":\"\u{a7}7.5 \u{2014} Anthropic: system message extracted to top-level 'system' field; messages array contains only non-system messages.\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-3\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":4096},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-3\",\"system\":\"Be helpful\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1074,7 +1384,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector104WireConformanceToRequestAnthropicToolWire() async throws { + func testVector135WireConformanceToRequestAnthropicToolWire() async throws { guard let vectorData = "{\"name\":\"anthropic_tool_wire\",\"description\":\"Anthropic: tools use input_schema (not parameters), name at top level\",\"stage\":\"wire\",\"provider\":\"anthropic\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"anthropic\"}},\"apiType\":\"chat\",\"model_id\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"claude-sonnet-4-20250514\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"max_tokens\":4096}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1084,7 +1394,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector105WireConformanceToRequestChatAudioMp3() async throws { + func testVector136WireConformanceToRequestChatAudioMp3() async throws { guard let vectorData = "{\"name\":\"chat_audio_mp3\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/mpeg MUST map to format 'mp3', not 'mpeg'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/mpeg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"mp3\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1094,7 +1404,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector106WireConformanceToRequestChatAudioPart() async throws { + func testVector137WireConformanceToRequestChatAudioPart() async throws { guard let vectorData = "{\"name\":\"chat_audio_part\",\"description\":\"\u{a7}7.1.2 \u{2014} AudioPart with mediaType audio/wav maps to input_audio with format 'wav'.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"audio\",\"value\":\"YmFzZTY0ZGF0YQ==\",\"mediaType\":\"audio/wav\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"YmFzZTY0ZGF0YQ==\",\"format\":\"wav\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1104,7 +1414,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector107WireConformanceToRequestChatImageBase64() async throws { + func testVector138WireConformanceToRequestChatImageBase64() async throws { guard let vectorData = "{\"name\":\"chat_image_base64\",\"description\":\"Base64 data URI image part preserved as-is in image_url\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this\"},{\"kind\":\"image\",\"value\":\"data:image/png;base64,iVBOR...\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBOR...\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1114,7 +1424,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector108WireConformanceToRequestChatImagePart() async throws { + func testVector139WireConformanceToRequestChatImagePart() async throws { guard let vectorData = "{\"name\":\"chat_image_part\",\"description\":\"Image content part converted to OpenAI image_url format\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What's in this image?\"},{\"kind\":\"image\",\"value\":\"https://example.com/photo.jpg\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What's in this image?\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://example.com/photo.jpg\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1124,7 +1434,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector109WireConformanceToRequestChatMultipartContent() async throws { + func testVector140WireConformanceToRequestChatMultipartContent() async throws { guard let vectorData = "{\"name\":\"chat_multipart_content\",\"description\":\"\u{a7}7.1.1/\u{a7}7.1.2 \u{2014} Message with TextPart + ImagePart MUST produce an array of typed content blocks.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Describe this image\"},{\"kind\":\"image\",\"value\":\"https://img.png\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe this image\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://img.png\"}}]}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1134,7 +1444,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector110WireConformanceToRequestChatSimple() async throws { + func testVector141WireConformanceToRequestChatSimple() async throws { guard let vectorData = "{\"name\":\"chat_simple\",\"description\":\"\u{a7}7.1.7 \u{2014} Two messages (system + user), no tools, no options. Minimal chat request.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"system\",\"content\":\"Be helpful\"},{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1144,7 +1454,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector111WireConformanceToRequestChatSingleTextOptimized() async throws { + func testVector142WireConformanceToRequestChatSingleTextOptimized() async throws { guard let vectorData = "{\"name\":\"chat_single_text_optimized\",\"description\":\"\u{a7}7.1.1 \u{2014} Message with exactly 1 TextPart MUST produce a plain string content, not an array.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is 2+2?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1154,7 +1464,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector112WireConformanceToRequestChatWithOptions() async throws { + func testVector143WireConformanceToRequestChatWithOptions() async throws { guard let vectorData = "{\"name\":\"chat_with_options\",\"description\":\"\u{a7}7.1.5 \u{2014} All standard ModelOptions mapped to OpenAI parameter names.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"temperature\":0.5,\"maxOutputTokens\":100,\"topP\":0.9,\"frequencyPenalty\":0.1,\"presencePenalty\":0.2,\"seed\":42,\"stopSequences\":[\"END\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"temperature\":0.5,\"max_completion_tokens\":100,\"top_p\":0.9,\"frequency_penalty\":0.1,\"presence_penalty\":0.2,\"seed\":42,\"stop\":[\"END\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1164,7 +1474,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector113WireConformanceToRequestEmbeddingWire() async throws { + func testVector144WireConformanceToRequestEmbeddingWire() async throws { guard let vectorData = "{\"name\":\"embedding_wire\",\"description\":\"\u{a7}7.2 \u{2014} Embedding request: text extracted from messages, single string input (not array).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"embedding\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"embedding\",\"model_id\":\"text-embedding-3-small\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello world\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"text-embedding-3-small\",\"input\":\"Hello world\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1174,7 +1484,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector114WireConformanceToRequestImageWire() async throws { + func testVector145WireConformanceToRequestImageWire() async throws { guard let vectorData = "{\"name\":\"image_wire\",\"description\":\"\u{a7}7.3 \u{2014} Image generation request: prompt extracted from last user message.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"image\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"image\",\"model_id\":\"dall-e-3\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"A cat\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"dall-e-3\",\"prompt\":\"A cat\"}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1184,7 +1494,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector115WireConformanceToRequestKindToJsonTypeMapping() async throws { + func testVector146WireConformanceToRequestKindToJsonTypeMapping() async throws { guard let vectorData = "{\"name\":\"kind_to_json_type_mapping\",\"description\":\"\u{a7}7.1.4 \u{2014} All Property kind values mapped to JSON Schema types: string\u{2192}string, integer\u{2192}integer, float\u{2192}number, boolean\u{2192}boolean, array\u{2192}array, object\u{2192}object.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Test\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":[{\"name\":\"a_string\",\"kind\":\"string\"},{\"name\":\"an_integer\",\"kind\":\"integer\"},{\"name\":\"a_float\",\"kind\":\"float\"},{\"name\":\"a_boolean\",\"kind\":\"boolean\"},{\"name\":\"an_array\",\"kind\":\"array\",\"items\":{\"kind\":\"string\"}},{\"name\":\"an_object\",\"kind\":\"object\"}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Test\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"test_types\",\"description\":\"Tests all type mappings\",\"parameters\":{\"type\":\"object\",\"properties\":{\"a_string\":{\"type\":\"string\"},\"an_integer\":{\"type\":\"integer\"},\"a_float\":{\"type\":\"number\"},\"a_boolean\":{\"type\":\"boolean\"},\"an_array\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"an_object\":{\"type\":\"object\"}}}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1194,7 +1504,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector116WireConformanceToRequestOptionsAdditionalProperties() async throws { + func testVector147WireConformanceToRequestOptionsAdditionalProperties() async throws { guard let vectorData = "{\"name\":\"options_additional_properties\",\"description\":\"\u{a7}7.1.5 \u{2014} additionalProperties from ModelOptions MUST be merged into the request as top-level keys.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"additionalProperties\":{\"logprobs\":true}},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"logprobs\":true}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1204,7 +1514,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector117WireConformanceToRequestOptionsMaxCompletionTokens() async throws { + func testVector148WireConformanceToRequestOptionsMaxCompletionTokens() async throws { guard let vectorData = "{\"name\":\"options_max_completion_tokens\",\"description\":\"\u{a7}7.1.5 \u{2014} maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"maxOutputTokens\":500},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_completion_tokens\":500}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1214,7 +1524,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector118WireConformanceToRequestOptionsStopSequences() async throws { + func testVector149WireConformanceToRequestOptionsStopSequences() async throws { guard let vectorData = "{\"name\":\"options_stop_sequences\",\"description\":\"\u{a7}7.1.5 \u{2014} stopSequences MUST map to 'stop' in the OpenAI wire format.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{\"stopSequences\":[\".\",\"!\"]},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"stop\":[\".\",\"!\"]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1224,7 +1534,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector119WireConformanceToRequestResponsesSimple() async throws { + func testVector150WireConformanceToRequestResponsesSimple() async throws { guard let vectorData = "{\"name\":\"responses_simple\",\"description\":\"Responses API: system becomes instructions, messages become input items\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"system\",\"content\":[{\"kind\":\"text\",\"value\":\"Be helpful\"}]},{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"instructions\":\"Be helpful\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1234,7 +1544,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector120WireConformanceToRequestResponsesStructuredOutput() async throws { + func testVector151WireConformanceToRequestResponsesStructuredOutput() async throws { guard let vectorData = "{\"name\":\"responses_structured_output\",\"description\":\"Responses API: structured output uses text.format.json_schema\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Give me data\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\"},{\"name\":\"temp\",\"kind\":\"integer\"}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Give me data\"}],\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"structured_output\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":[\"string\",\"null\"]},\"temp\":{\"type\":[\"integer\",\"null\"]}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false},\"strict\":true}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1244,7 +1554,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector121WireConformanceToRequestResponsesWithTools() async throws { + func testVector152WireConformanceToRequestResponsesWithTools() async throws { guard let vectorData = "{\"name\":\"responses_with_tools\",\"description\":\"Responses API: tools use flat format (no nested function key)\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"responses\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"responses\",\"model_id\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Weather?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"kind\":\"function\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o\",\"input\":[{\"role\":\"user\",\"content\":\"Weather?\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1254,7 +1564,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector122WireConformanceToRequestStructuredOutput() async throws { + func testVector153WireConformanceToRequestStructuredOutput() async throws { guard let vectorData = "{\"name\":\"structured_output\",\"description\":\"\u{a7}7.1.6 \u{2014} outputs converted to response_format with json_schema, strict=true, additionalProperties=false.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather in Paris?\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true},{\"name\":\"temp\",\"kind\":\"integer\",\"required\":true}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"temp\":{\"type\":\"integer\"}},\"required\":[\"city\",\"temp\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1264,7 +1574,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector123WireConformanceToRequestStructuredOutputNestedOptional() async throws { + func testVector154WireConformanceToRequestStructuredOutputNestedOptional() async throws { guard let vectorData = "{\"name\":\"structured_output_nested_optional\",\"description\":\"\u{a7}7.1.4/\u{a7}7.1.6 \u{2014} OpenAI strict mode recursively requires every object property and represents optional fields as nullable.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Choose a visual style.\"}]}],\"tools\":[],\"options\":{},\"outputs\":[{\"name\":\"style\",\"kind\":\"object\",\"required\":true,\"properties\":[{\"name\":\"color\",\"kind\":\"string\",\"required\":true},{\"name\":\"border\",\"kind\":\"string\",\"required\":false}]}]},\"expected\":{\"request_body\":{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Choose a visual style.\"}],\"response_format\":{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"structured_output\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"style\":{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"string\"},\"border\":{\"type\":[\"string\",\"null\"]}},\"required\":[\"color\",\"border\"],\"additionalProperties\":false}},\"required\":[\"style\"],\"additionalProperties\":false}}}}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1274,7 +1584,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector124WireConformanceToRequestToolsBindingsStripped() async throws { + func testVector155WireConformanceToRequestToolsBindingsStripped() async throws { guard let vectorData = "{\"name\":\"tools_bindings_stripped\",\"description\":\"\u{a7}7.1.3 \u{2014} Parameters listed in bindings MUST be stripped from wire tools (properties AND required).\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"location\",\"kind\":\"string\",\"required\":true},{\"name\":\"unit\",\"kind\":\"string\",\"required\":true}],\"bindings\":{\"unit\":{\"input\":\"preferred_unit\"}}}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\"}},\"required\":[\"location\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1284,7 +1594,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector125WireConformanceToRequestToolsFunctionWire() async throws { + func testVector156WireConformanceToRequestToolsFunctionWire() async throws { guard let vectorData = "{\"name\":\"tools_function_wire\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool projected as OpenAI function definition with JSON Schema parameters.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"What is the weather?\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1294,7 +1604,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector126WireConformanceToRequestToolsNullWhenEmpty() async throws { + func testVector157WireConformanceToRequestToolsNullWhenEmpty() async throws { guard let vectorData = "{\"name\":\"tools_null_when_empty\",\"description\":\"\u{a7}7.1.3 \u{2014} When no tools are defined, the tools key MUST be absent from the request entirely.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Hello\"}]}],\"tools\":[],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { @@ -1304,7 +1614,7 @@ final class VectorConformanceTests: XCTestCase { _ = await VectorRunner.runVector(contract: "WireConformance", operation: "toRequest", vector: vector, sync: false, seam: seam()) } - func testVector127WireConformanceToRequestToolsStrictMode() async throws { + func testVector158WireConformanceToRequestToolsStrictMode() async throws { guard let vectorData = "{\"name\":\"tools_strict_mode\",\"description\":\"\u{a7}7.1.3 \u{2014} FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.\",\"stage\":\"wire\",\"provider\":\"openai\",\"targetApi\":\"chat\",\"input\":{\"agent\":{\"model\":{\"provider\":\"openai\"}},\"apiType\":\"chat\",\"model_id\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":[{\"kind\":\"text\",\"value\":\"Get the weather\"}]}],\"tools\":[{\"kind\":\"function\",\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":[{\"name\":\"city\",\"kind\":\"string\",\"required\":true}]}],\"options\":{},\"outputs\":[]},\"expected\":{\"request_body\":{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"Get the weather\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get weather\",\"strict\":true,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}]}},\"operation\":\"toRequest\"}".data(using: .utf8), let vector = try JSONSerialization.jsonObject(with: vectorData) as? [String: Any] else { diff --git a/runtime/swift/prompty-model/model_capabilities.json b/runtime/swift/prompty-model/model_capabilities.json new file mode 100644 index 000000000..6951f344f --- /dev/null +++ b/runtime/swift/prompty-model/model_capabilities.json @@ -0,0 +1,60 @@ +{ + "description": "Cross-runtime fallback capability data for provider model discovery. Some provider /models endpoints (Anthropic, Foundry) return capability fields directly; others (OpenAI) return only ids. To keep discovery results consistent across providers AND across runtimes, every Prompty runtime embeds THIS file and applies one shared rule: provider-supplied fields always win; entries here only fill fields the provider left empty (fill-only-missing). Model ids are matched by longest prefix within a provider's list. This dataset is deliberately NOT emitted from TypeSpec: it is volatile provider data (context windows, modalities, new model families) refreshed as a snapshot, whereas TypeSpec owns the structural ModelInfo contract. Fields use the canonical camelCase ModelInfo names. A missing 'contextWindow' means unknown; a present empty modality list (e.g. []) is intentional (e.g. embeddings produce no textual/image output modality).", + "match": "longest_prefix", + "providers": { + "openai": [ + { + "prefix": "gpt-4o-mini", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4o", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4-turbo", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4", + "contextWindow": 8192, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-3.5-turbo", + "contextWindow": 16385, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "text-embedding-3-small", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "text-embedding-3-large", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "gpt-image-1", + "inputModalities": ["text", "image"], + "outputModalities": ["image"] + }, + { + "prefix": "dall-e-3", + "inputModalities": ["text"], + "outputModalities": ["image"] + } + ] + } +} diff --git a/runtime/swift/prompty/Info.plist b/runtime/swift/prompty/Info.plist new file mode 100644 index 000000000..3eaf43945 --- /dev/null +++ b/runtime/swift/prompty/Info.plist @@ -0,0 +1,16 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleIdentifier + prompty.Prompty.resources + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Prompty_Prompty + CFBundlePackageType + BNDL + + diff --git a/runtime/swift/prompty/Sources/Prompty/EndpointTrust.swift b/runtime/swift/prompty/Sources/Prompty/EndpointTrust.swift new file mode 100644 index 000000000..5daceaa9a --- /dev/null +++ b/runtime/swift/prompty/Sources/Prompty/EndpointTrust.swift @@ -0,0 +1,29 @@ +import Foundation +import PromptyModel + +/// Loader-side wiring for endpoint trust (spec §2.5.1). +/// +/// The classification and origin matching live in `PromptyModel.EndpointTrust`; +/// this extracts the RAW `model.connection` (following whole-value `${file:}` +/// references for `model` and `model.connection` with the same containment as +/// reference resolution) and maps a rejection onto `LoadError.untrustedEndpoint`. +enum EndpointTrustCheck { + + static func check( + _ data: [String: Any], agentDirectory: URL, allowedRoots: [URL], trustedEndpoints: [String] + ) throws { + let connection = try EndpointTrust.rawConnection(data) { reference -> Any? in + let target = URL(fileURLWithPath: reference, relativeTo: agentDirectory) + .standardizedFileURL.resolvingSymlinksInPath() + // A missing file is reported by reference resolution, after this check. + guard FileManager.default.fileExists(atPath: target.path) else { return nil } + return try References.resolveString( + "${file:\(reference)}", key: "model", agentDirectory: agentDirectory, allowedRoots: allowedRoots) + } + do { + try EndpointTrust.checkConnection(connection, trustedEndpoints: trustedEndpoints) + } catch let error as PromptyLoadError { + throw LoadError.untrustedEndpoint(error.message) + } + } +} diff --git a/runtime/swift/prompty/Sources/Prompty/Errors.swift b/runtime/swift/prompty/Sources/Prompty/Errors.swift index dba5c6fef..114b19e51 100644 --- a/runtime/swift/prompty/Sources/Prompty/Errors.swift +++ b/runtime/swift/prompty/Sources/Prompty/Errors.swift @@ -19,6 +19,9 @@ public enum LoadError: Error, CustomStringConvertible, Equatable { /// value that can never be a property. `path` is the full dotted path to the /// offending entry and `valueCategory` names the rejected shape. case invalidNamedCollectionEntry(path: String, valueCategory: String) + /// The connection would send ambient credentials to a destination the file + /// chose that the host has not trusted (spec §2.5.1). + case untrustedEndpoint(String) public var description: String { switch self { @@ -37,6 +40,8 @@ public enum LoadError: Error, CustomStringConvertible, Equatable { "invalid-named-collection-entry: \(path) holds a \(valueCategory), " + "which is not a valid named collection entry. Declare the array inside " + "a property instead, for example `\(path): { kind: array, default: [...] }`" + case .untrustedEndpoint(let message): + return message } } } diff --git a/runtime/swift/prompty/Sources/Prompty/Loader.swift b/runtime/swift/prompty/Sources/Prompty/Loader.swift index 36d801746..a72467734 100644 --- a/runtime/swift/prompty/Sources/Prompty/Loader.swift +++ b/runtime/swift/prompty/Sources/Prompty/Loader.swift @@ -12,8 +12,14 @@ public struct LoadOptions { /// file's own directory is always permitted. public var allowedFileRoots: [URL] - public init(allowedFileRoots: [URL] = []) { + /// Extra origins (e.g. `https://my-proxy.contoso.com`, `https://*.contoso.com`, + /// or `*`) trusted to receive ambient credentials, in addition to the built-in + /// provider origins and `PROMPTY_TRUSTED_ENDPOINTS` (spec §2.5.1). + public var trustedEndpoints: [String] + + public init(allowedFileRoots: [URL] = [], trustedEndpoints: [String] = []) { self.allowedFileRoots = allowedFileRoots + self.trustedEndpoints = trustedEndpoints } public static let `default` = LoadOptions() @@ -84,8 +90,13 @@ public enum Loader { // 3. A `.prompty` file always describes a prompt. data["kind"] = Defaults.kind - // 4. Resolve ${env:} / ${file:} before handing the tree to the model. + // 4. Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. let agentDirectory = filePath.deletingLastPathComponent() + try EndpointTrustCheck.check( + data, agentDirectory: agentDirectory, allowedRoots: options.allowedFileRoots, + trustedEndpoints: options.trustedEndpoints) + + // 5. Resolve ${env:} / ${file:} before handing the tree to the model. var tree: Any = data try References.resolve( &tree, agentDirectory: agentDirectory, allowedRoots: options.allowedFileRoots) @@ -94,12 +105,12 @@ public enum Loader { throw LoadError.invalidFrontmatter("Frontmatter must be a YAML mapping") } - // 5. Normalize the shapes the schema accepts as shorthand. + // 6. Normalize the shapes the schema accepts as shorthand. normalizeModel(&resolved) try normalizeProperties(&resolved, key: "inputs", path: "inputs") try normalizeProperties(&resolved, key: "outputs", path: "outputs") - // 6. Hand off to the generated model — the canonical type layer. + // 7. Hand off to the generated model — the canonical type layer. var agent: Agent do { agent = try Agent.load(resolved) @@ -107,7 +118,7 @@ public enum Loader { throw LoadError.invalidModel(String(describing: error)) } - // 7. Record the source path so relative tool references can resolve later. + // 8. Record the source path so relative tool references can resolve later. var metadata = agent.metadata ?? [:] metadata[Defaults.sourcePathKey] = filePath.path agent.metadata = metadata diff --git a/runtime/swift/prompty/Sources/PromptyAnthropic/AnthropicConfig.swift b/runtime/swift/prompty/Sources/PromptyAnthropic/AnthropicConfig.swift index 9e651fb53..395102ebe 100644 --- a/runtime/swift/prompty/Sources/PromptyAnthropic/AnthropicConfig.swift +++ b/runtime/swift/prompty/Sources/PromptyAnthropic/AnthropicConfig.swift @@ -27,7 +27,10 @@ struct AnthropicConfig { if let connection { let fields = (try? connection.save()) ?? [:] if let value = fields["endpoint"] as? String, !value.isEmpty { endpoint = value } - if let value = fields["apiKey"] as? String, !value.isEmpty { apiKey = value } + // apiKey is withheld from save(); read the live value from the typed connection. + if case .apiKeyConnection(let keyed) = connection, !keyed.apiKey.isEmpty { + apiKey = keyed.apiKey + } } if endpoint.isEmpty { diff --git a/runtime/swift/prompty/Sources/PromptyOpenAI/OpenAIConfig.swift b/runtime/swift/prompty/Sources/PromptyOpenAI/OpenAIConfig.swift index 691075061..777663715 100644 --- a/runtime/swift/prompty/Sources/PromptyOpenAI/OpenAIConfig.swift +++ b/runtime/swift/prompty/Sources/PromptyOpenAI/OpenAIConfig.swift @@ -28,7 +28,10 @@ struct OpenAIConfig { if let connection { let fields = connectionFields(connection) if let value = fields["endpoint"] as? String, !value.isEmpty { endpoint = value } - if let value = fields["apiKey"] as? String, !value.isEmpty { apiKey = value } + // apiKey is withheld from save(); read the live value from the typed connection. + if case .apiKeyConnection(let keyed) = connection, !keyed.apiKey.isEmpty { + apiKey = keyed.apiKey + } if let value = fields["headers"] as? [String: Any] { for (key, header) in value { headers[key] = JSONSupport.stringify(header) } } diff --git a/runtime/swift/prompty/Tests/PromptyTests/ConnectionKeyTests.swift b/runtime/swift/prompty/Tests/PromptyTests/ConnectionKeyTests.swift new file mode 100644 index 000000000..fbf28932a --- /dev/null +++ b/runtime/swift/prompty/Tests/PromptyTests/ConnectionKeyTests.swift @@ -0,0 +1,49 @@ +import Foundation +import PromptyModel +import XCTest + +@testable import Prompty +@testable import PromptyAnthropic +@testable import PromptyOpenAI + +/// A file-supplied `apiKey` must be the key sent, never the ambient env key +/// (`apiKey` is withheld from `save()`, so configs must read the live value). +final class ConnectionKeyTests: XCTestCase { + + private func agent(provider: String) throws -> Agent { + try Agent.load([ + "kind": "prompt", + "name": "connection-key", + "model": [ + "id": "m", + "provider": provider, + "connection": [ + "kind": "key", + "endpoint": "https://custom.example.com", + "apiKey": "file-key", + ], + ], + ]) + } + + private var savedEnv: [String: String?] = [:] + + override func setUp() { + for key in ["ANTHROPIC_API_KEY", "OPENAI_API_KEY"] { + savedEnv[key] = ProcessInfo.processInfo.environment[key] + Env.set(key, "ambient-key") + } + } + + override func tearDown() { + for (key, value) in savedEnv { Env.set(key, value) } + } + + func testAnthropicUsesFileKey() throws { + XCTAssertEqual(try AnthropicConfig.resolve(agent(provider: "anthropic")).apiKey, "file-key") + } + + func testOpenAIUsesFileKey() throws { + XCTAssertEqual(try OpenAIConfig.resolve(agent(provider: "openai")).apiKey, "file-key") + } +} diff --git a/runtime/swift/prompty/Tests/PromptyTests/EndpointTrustTests.swift b/runtime/swift/prompty/Tests/PromptyTests/EndpointTrustTests.swift new file mode 100644 index 000000000..1c68da592 --- /dev/null +++ b/runtime/swift/prompty/Tests/PromptyTests/EndpointTrustTests.swift @@ -0,0 +1,202 @@ +import Foundation +import PromptyModel +import XCTest + +@testable import Prompty + +/// Endpoint trust (spec §2.5.1): ambient credentials only go to host-approved destinations. +final class EndpointTrustTests: XCTestCase { + + private var directory: URL! + private var savedEnv: [String: String?] = [:] + + override func setUpWithError() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("prompty_trust_\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + setEnv(EndpointTrust.trustedEndpointsEnv, nil) + } + + override func tearDownWithError() throws { + for (key, value) in savedEnv { Env.set(key, value) } + savedEnv = [:] + try? FileManager.default.removeItem(at: directory) + } + + private func setEnv(_ key: String, _ value: String?) { + if !savedEnv.keys.contains(key) { + savedEnv.updateValue(ProcessInfo.processInfo.environment[key], forKey: key) + } + Env.set(key, value) + } + + @discardableResult + private func write(_ connection: String, name: String = "t.prompty") throws -> String { + let path = directory.appendingPathComponent(name) + try "---\nname: t\nmodel:\n id: gpt-4\n connection:\n\(connection)\n---\nHello\n" + .write(to: path, atomically: true, encoding: .utf8) + return path.path + } + + private func assertUntrusted( + _ body: @autoclosure () throws -> Agent, contains fragment: String = "not trusted", + file: StaticString = #filePath, line: UInt = #line + ) { + XCTAssertThrowsError(try body(), file: file, line: line) { error in + guard case .untrustedEndpoint(let message) = error as? LoadError else { + return XCTFail("expected LoadError.untrustedEndpoint, got \(error)", file: file, line: line) + } + XCTAssertTrue(message.contains(fragment), "'\(message)' lacks '\(fragment)'", file: file, line: line) + } + } + + private func endpoint(_ agent: Agent) -> String? { + switch agent.model?.connection { + case .apiKeyConnection(let c): return c.endpoint + case .foundryConnection(let c): return c.endpoint + default: return nil + } + } + + // MARK: - isTrusted + + func testBuiltinTrusted() { + for url in [ + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + ] { + XCTAssertTrue(EndpointTrust.isTrusted(url), url) + } + } + + func testUntrusted() { + for url in [ + "http://api.openai.com", + "https://api.openai.com:8443", + "https://openai.azure.com", + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "", + ] { + XCTAssertFalse(EndpointTrust.isTrusted(url), url) + } + } + + func testHostOptionAndWildcard() { + XCTAssertTrue(EndpointTrust.isTrusted("http://localhost:11434/v1", trustedEndpoints: ["http://LOCALHOST:11434"])) + XCTAssertTrue(EndpointTrust.isTrusted("https://x.contoso.com", trustedEndpoints: ["https://*.contoso.com"])) + XCTAssertFalse(EndpointTrust.isTrusted("https://contoso.com", trustedEndpoints: ["https://*.contoso.com"])) + } + + func testEnvListAndStar() { + setEnv(EndpointTrust.trustedEndpointsEnv, " https://a.example.com , http://localhost:8080") + XCTAssertTrue(EndpointTrust.isTrusted("http://localhost:8080/x")) + XCTAssertFalse(EndpointTrust.isTrusted("http://localhost:8081")) + setEnv(EndpointTrust.trustedEndpointsEnv, "*") + XCTAssertTrue(EndpointTrust.isTrusted("http://anything.example")) + } + + // MARK: - Loader enforcement + + func testAmbientLiteralEndpointRejected() throws { + let path = try write(" kind: foundry\n endpoint: http://127.0.0.1:9999") + assertUntrusted(try Loader.load(path: path), contains: "127.0.0.1") + } + + func testEnvKeyLiteralEndpointRejected() throws { + setEnv("SOME_KEY", "secret") + let path = try write(" kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}") + assertUntrusted(try Loader.load(path: path)) + } + + func testOAuthTokenUrlChecked() throws { + setEnv("SECRET", "s") + let path = try write( + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token") + assertUntrusted(try Loader.load(path: path), contains: "tokenUrl") + } + + func testMismatchedCredentialFieldRejected() throws { + let foundry = try write(" kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk") + assertUntrusted(try Loader.load(path: foundry)) + let key = try write(" kind: key\n endpoint: https://evil.example.com\n clientSecret: junk") + assertUntrusted(try Loader.load(path: key)) + } + + func testLiteralKeyAllowed() throws { + let path = try write(" kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine") + XCTAssertEqual(endpoint(try Loader.load(path: path)), "http://127.0.0.1:9999") + } + + func testEnvEndpointAllowed() throws { + setEnv("EP", "http://127.0.0.1:9999") + let path = try write(" kind: foundry\n endpoint: ${env:EP}") + XCTAssertEqual(endpoint(try Loader.load(path: path)), "http://127.0.0.1:9999") + } + + func testEnvDefaultUsedRejected() throws { + setEnv("UNSET_EP", nil) + let path = try write(" kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}") + assertUntrusted(try Loader.load(path: path)) + } + + func testTrustedOptionAllows() throws { + let path = try write(" kind: foundry\n endpoint: http://127.0.0.1:9999/api") + let agent = try Loader.load(path: path, options: LoadOptions(trustedEndpoints: ["http://127.0.0.1:9999"])) + XCTAssertEqual(endpoint(agent), "http://127.0.0.1:9999/api") + } + + func testAsyncHonorsOption() async throws { + let path = try write(" kind: foundry\n endpoint: http://127.0.0.1:9999") + do { + _ = try await Loader.loadAsync(path: path) + XCTFail("expected untrusted_endpoint") + } catch LoadError.untrustedEndpoint { + } + let agent = try await Loader.loadAsync( + path: path, options: LoadOptions(trustedEndpoints: ["http://127.0.0.1:9999"])) + XCTAssertEqual(endpoint(agent), "http://127.0.0.1:9999") + } + + func testFileModelReferenceChecked() throws { + let model: [String: Any] = [ + "id": "gpt-4", "connection": ["kind": "foundry", "endpoint": "https://evil.example.com"], + ] + try JSONSerialization.data(withJSONObject: model).write(to: directory.appendingPathComponent("model.json")) + let path = directory.appendingPathComponent("t.prompty") + try "---\nname: t\nmodel: ${file:model.json}\n---\nHi\n".write(to: path, atomically: true, encoding: .utf8) + assertUntrusted(try Loader.load(path: path.path)) + } + + func testFileTraversalStillReportsFileReference() throws { + let app = directory.appendingPathComponent("app") + try FileManager.default.createDirectory(at: app, withIntermediateDirectories: true) + let secret: [String: Any] = ["kind": "foundry", "endpoint": "https://evil.example.com"] + try JSONSerialization.data(withJSONObject: secret).write(to: directory.appendingPathComponent("secret.json")) + let path = app.appendingPathComponent("t.prompty") + try "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:../secret.json}\n---\nHi\n" + .write(to: path, atomically: true, encoding: .utf8) + XCTAssertThrowsError(try Loader.load(path: path.path)) { error in + guard case .fileReference = error as? LoadError else { + return XCTFail("expected LoadError.fileReference, got \(error)") + } + } + } + + func testFrontmatterCannotSelfTrust() throws { + let path = directory.appendingPathComponent("t.prompty") + try ( + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n" + ).write(to: path, atomically: true, encoding: .utf8) + assertUntrusted(try Loader.load(path: path.path)) + } +} diff --git a/runtime/swift/prompty/Tests/PromptyTests/LoadVectorTests.swift b/runtime/swift/prompty/Tests/PromptyTests/LoadVectorTests.swift index 413c5a570..a4e10dfb3 100644 --- a/runtime/swift/prompty/Tests/PromptyTests/LoadVectorTests.swift +++ b/runtime/swift/prompty/Tests/PromptyTests/LoadVectorTests.swift @@ -142,7 +142,11 @@ final class LoadVectorTests: XCTestCase { private static func runVector( name: String, input: [String: Any], expected: [String: Any], expectedError: [String: Any]? ) throws { - let env = input["env"] as? [String: Any] ?? [:] + var env = input["env"] as? [String: Any] ?? [:] + // Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + if env[EndpointTrust.trustedEndpointsEnv] == nil { + env[EndpointTrust.trustedEndpointsEnv] = NSNull() + } return try withEnvironment(env) { // Vectors that declare a canonical `expectedError` drive the loader (or @@ -167,8 +171,11 @@ final class LoadVectorTests: XCTestCase { /// Build a `Agent` from whichever input form the vector uses. static func loadAgent(_ input: [String: Any]) throws -> Agent { + let options = LoadOptions( + trustedEndpoints: (input["trustedEndpoints"] as? [Any] ?? []).compactMap { $0 as? String }) + if let fixture = input["fixture"] as? String { - return try Loader.load(path: Spec.fixtures.appendingPathComponent(fixture).path) + return try Loader.load(path: Spec.fixtures.appendingPathComponent(fixture).path, options: options) } if let files = input["files"] as? [String: Any] { @@ -203,18 +210,19 @@ final class LoadVectorTests: XCTestCase { let raw = try promptyDocument(frontmatter: input["frontmatter"]) return try Loader.load( - contents: raw, basePath: agentDir.appendingPathComponent("virtual.prompty").path) + contents: raw, basePath: agentDir.appendingPathComponent("virtual.prompty").path, + options: options) } if let raw = input["frontmatter_raw"] as? String { - return try Loader.load(contents: raw, basePath: workingFile) + return try Loader.load(contents: raw, basePath: workingFile, options: options) } guard input["frontmatter"] != nil else { throw VectorFailure("vector has no fixture, frontmatter, or frontmatter_raw") } let raw = try promptyDocument(frontmatter: input["frontmatter"]) - return try Loader.load(contents: raw, basePath: workingFile) + return try Loader.load(contents: raw, basePath: workingFile, options: options) } /// Serialize a vector's `frontmatter` object into a `.prompty` document. @@ -324,6 +332,7 @@ final class LoadVectorTests: XCTestCase { case .invalidFrontmatter: return ["kind": "invalid_frontmatter"] case .invalidModel: return ["kind": "invalid_model"] case .invalidNamedCollectionEntry: return ["kind": "invalid_named_collection_entry"] + case .untrustedEndpoint: return ["kind": "untrusted_endpoint"] } } @@ -642,8 +651,10 @@ final class LoadVectorTests: XCTestCase { func withEnvironment(_ values: [String: Any], _ body: () throws -> T) rethrows -> T { var restore: [String: String?] = [:] for (key, value) in values { - guard let string = value as? String else { continue } - restore[key] = ProcessInfo.processInfo.environment[key] + // `NSNull` unsets the variable for the duration of `body`. + let string = value as? String + guard string != nil || value is NSNull else { continue } + restore.updateValue(ProcessInfo.processInfo.environment[key], forKey: key) Env.set(key, string) } defer { diff --git a/runtime/swift/prompty/Tests/PromptyTests/ReadmeSnippetTests.swift b/runtime/swift/prompty/Tests/PromptyTests/ReadmeSnippetTests.swift index 0dd50bbde..8f2f61a72 100644 --- a/runtime/swift/prompty/Tests/PromptyTests/ReadmeSnippetTests.swift +++ b/runtime/swift/prompty/Tests/PromptyTests/ReadmeSnippetTests.swift @@ -33,6 +33,13 @@ final class ReadmeSnippetTests: XCTestCase { return raw } + func trusted() throws -> Agent { + try Loader.load( + path: "local.prompty", + options: LoadOptions(trustedEndpoints: ["http://localhost:11434", "https://*.contoso.com"]) + ) + } + func toolLoop( agent: Agent, raw: Any?, inputs: [String: Any], myTools: [String: ([String: Any]) throws -> String] @@ -48,6 +55,7 @@ final class ReadmeSnippetTests: XCTestCase { XCTAssertNotNil(quickStart) XCTAssertNotNil(stages) + XCTAssertNotNil(trusted) XCTAssertNotNil(toolLoop) } } diff --git a/runtime/swift/prompty/model_capabilities.json b/runtime/swift/prompty/model_capabilities.json new file mode 100644 index 000000000..6951f344f --- /dev/null +++ b/runtime/swift/prompty/model_capabilities.json @@ -0,0 +1,60 @@ +{ + "description": "Cross-runtime fallback capability data for provider model discovery. Some provider /models endpoints (Anthropic, Foundry) return capability fields directly; others (OpenAI) return only ids. To keep discovery results consistent across providers AND across runtimes, every Prompty runtime embeds THIS file and applies one shared rule: provider-supplied fields always win; entries here only fill fields the provider left empty (fill-only-missing). Model ids are matched by longest prefix within a provider's list. This dataset is deliberately NOT emitted from TypeSpec: it is volatile provider data (context windows, modalities, new model families) refreshed as a snapshot, whereas TypeSpec owns the structural ModelInfo contract. Fields use the canonical camelCase ModelInfo names. A missing 'contextWindow' means unknown; a present empty modality list (e.g. []) is intentional (e.g. embeddings produce no textual/image output modality).", + "match": "longest_prefix", + "providers": { + "openai": [ + { + "prefix": "gpt-4o-mini", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4o", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4-turbo", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4", + "contextWindow": 8192, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-3.5-turbo", + "contextWindow": 16385, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "text-embedding-3-small", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "text-embedding-3-large", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "gpt-image-1", + "inputModalities": ["text", "image"], + "outputModalities": ["image"] + }, + { + "prefix": "dall-e-3", + "inputModalities": ["text"], + "outputModalities": ["image"] + } + ] + } +} diff --git a/runtime/typescript/CHANGELOG.md b/runtime/typescript/CHANGELOG.md index 7f231be67..0af4aeb3d 100644 --- a/runtime/typescript/CHANGELOG.md +++ b/runtime/typescript/CHANGELOG.md @@ -2,7 +2,12 @@ All notable changes to the Prompty TypeScript runtime will be documented here. -## [2.0.1] — 2026-10-16 +## [Unreleased] + +### Security +- `load()` enforces spec §2.5.1 endpoint trust: a file whose connection uses ambient credentials may not target an untrusted, file-chosen `endpoint`/`tokenUrl` (`untrusted_endpoint` load error). Add origins via the `trustedEndpoints` load option or `PROMPTY_TRUSTED_ENDPOINTS`. + +## [2.0.1] � 2026-10-16 ### Fixed - OpenAI Responses processor now falls back to nested completed message text when top-level `output_text` is missing or empty (#571) diff --git a/runtime/typescript/packages/core/README.md b/runtime/typescript/packages/core/README.md index 4317f4fdb..fcf45d27f 100644 --- a/runtime/typescript/packages/core/README.md +++ b/runtime/typescript/packages/core/README.md @@ -75,6 +75,26 @@ user: Hello, my name is {{name}}. {{question}} ``` +## Endpoint Trust + +A `.prompty` file is untrusted input. When a connection relies on ambient +credentials (`DefaultAzureCredential`, `OPENAI_API_KEY`, …) rather than a +literal key in the file, `load()` refuses a file-chosen `endpoint` / `tokenUrl` +whose origin is not trusted (error kind `untrusted_endpoint`). Endpoints set via +`${env:VAR}` (with `VAR` set) are host-controlled and always allowed. + +Trusted origins are the built-in provider origins (`https://api.openai.com`, +`https://*.openai.azure.com`, …), plus any you add: + +```typescript +const agent = load("./local.prompty", { + trustedEndpoints: ["http://localhost:11434", "https://*.contoso.com"], +}); +``` + +or via `PROMPTY_TRUSTED_ENDPOINTS=http://localhost:11434,https://*.contoso.com`. +Use `*` to trust every endpoint. + ## Tracing ```typescript diff --git a/runtime/typescript/packages/core/src/core/endpoint-trust.ts b/runtime/typescript/packages/core/src/core/endpoint-trust.ts new file mode 100644 index 000000000..cb9f00bc3 --- /dev/null +++ b/runtime/typescript/packages/core/src/core/endpoint-trust.ts @@ -0,0 +1,167 @@ +/** + * Endpoint trust — keep ambient credentials on host-approved destinations (spec §2.5.1). + * + * A `.prompty` file is untrusted input. When its connection carries no + * file-supplied credential, executors fall back to ambient credentials + * (`DefaultAzureCredential`, `*_API_KEY` env vars). This module decides + * whether the file-chosen destination may receive them. + * + * @module + */ + +import { PromptyLoadError } from "./errors.js"; + +/** Env var holding additional comma-separated trusted origins. */ +export const TRUSTED_ENDPOINTS_ENV = "PROMPTY_TRUSTED_ENDPOINTS"; + +/** First-party provider origins that are always trusted. */ +export const BUILTIN_TRUSTED_ENDPOINTS: readonly string[] = Object.freeze([ + "https://api.openai.com", + "https://api.anthropic.com", + "https://*.openai.azure.com", + "https://*.services.ai.azure.com", + "https://*.cognitiveservices.azure.com", + "https://login.microsoftonline.com", +]); + +const DEFAULT_PORTS: Record = { http: 80, https: 443 }; +const CREDENTIAL_KEYS: Record = { + key: "apiKey", +}; +const DESTINATION_KEYS = ["endpoint", "tokenUrl"] as const; + +type Origin = [scheme: string, host: string, port: number]; + +/** Return `[scheme, host, port]` for an absolute http(s) URL, else `undefined`. */ +function origin(url: string): Origin | undefined { + const trimmed = url.trim(); + const match = /^([a-zA-Z][a-zA-Z0-9+.-]*):\/\//.exec(trimmed); + if (!match) return undefined; + const scheme = match[1].toLowerCase(); + if (!(scheme in DEFAULT_PORTS)) return undefined; + let parsed: URL; + try { + parsed = new URL(trimmed); + } catch { + return undefined; + } + const host = parsed.hostname.toLowerCase(); + if (!host) return undefined; + const port = parsed.port ? Number(parsed.port) : DEFAULT_PORTS[scheme]; + return [scheme, host, port]; +} + +function trustedEntries(trustedEndpoints?: Iterable): string[] { + const entries: unknown[] = [ + ...BUILTIN_TRUSTED_ENDPOINTS, + ...(trustedEndpoints ?? []), + ...(process.env[TRUSTED_ENDPOINTS_ENV] ?? "").split(","), + ]; + return entries + .filter((e): e is string => typeof e === "string") + .map((e) => e.trim()) + .filter((e) => e.length > 0); +} + +/** + * Return `true` if `url`'s origin matches a built-in, host-supplied, or + * `PROMPTY_TRUSTED_ENDPOINTS` trusted origin. + */ +export function isTrustedEndpoint( + url: string, + trustedEndpoints?: Iterable, +): boolean { + const entries = trustedEntries(trustedEndpoints); + if (entries.includes("*")) return true; + const target = origin(url); + if (!target) return false; + const [scheme, host, port] = target; + for (const entry of entries) { + const allowed = origin(entry); + if (!allowed || allowed[0] !== scheme || allowed[2] !== port) continue; + const pattern = allowed[1]; + if (pattern.startsWith("*.")) { + if (host.endsWith(pattern.slice(1))) return true; + } else if (pattern === host) { + return true; + } + } + return false; +} + +function isReference(value: string): boolean { + return value.startsWith("${") && value.endsWith("}"); +} + +function fileSuppliedCredential(connection: Record): boolean { + // Only the credential field the declared kind actually consumes counts; any other kind is ambient. + const kind = connection["kind"]; + const key = + typeof kind === "string" && + Object.prototype.hasOwnProperty.call(CREDENTIAL_KEYS, kind) + ? CREDENTIAL_KEYS[kind] + : undefined; + if (key === undefined) return false; + const value = connection[key]; + // Blank keys count as absent: executors that trim fall back to ambient credentials. + return ( + typeof value === "string" && value.trim().length > 0 && !isReference(value) + ); +} + +/** Return the effective destination if the file chose it, else `undefined`. */ +function fileControlledDestination(value: unknown): string | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value !== "string") return String(value); + if (isReference(value)) { + const inner = value.slice(2, -1); + const colon = inner.indexOf(":"); + const protocol = colon === -1 ? inner : inner.slice(0, colon); + const rest = colon === -1 ? "" : inner.slice(colon + 1); + if (protocol.toLowerCase() === "env") { + const next = rest.indexOf(":"); + const varName = next === -1 ? rest : rest.slice(0, next); + const defaultVal = next === -1 ? "" : rest.slice(next + 1); + if (process.env[varName] !== undefined) return undefined; + return defaultVal || undefined; + } + } + return value; +} + +/** + * Throw `untrusted_endpoint` if ambient credentials could reach a file-chosen, + * untrusted destination. + * + * @param connection - The RAW (unresolved) `model.connection` mapping. + * @param trustedEndpoints - Host-supplied trusted origins. + */ +export function checkConnectionTrust( + connection: unknown, + trustedEndpoints?: Iterable, +): void { + if ( + typeof connection !== "object" || + connection === null || + Array.isArray(connection) + ) { + return; + } + const conn = connection as Record; + if (conn["kind"] === "reference") return; + if (fileSuppliedCredential(conn)) return; + for (const key of DESTINATION_KEYS) { + const destination = fileControlledDestination(conn[key]); + if ( + destination !== undefined && + !isTrustedEndpoint(destination, trustedEndpoints) + ) { + throw new PromptyLoadError( + "untrusted_endpoint", + `Connection ${key} '${destination}' is not trusted to receive ambient credentials. ` + + `Set the ${key} via \${env:VAR}, add its origin to trustedEndpoints or ` + + `${TRUSTED_ENDPOINTS_ENV}, or use 'kind: reference'.`, + ); + } + } +} diff --git a/runtime/typescript/packages/core/src/core/errors.ts b/runtime/typescript/packages/core/src/core/errors.ts index 31ed71547..f954860eb 100644 --- a/runtime/typescript/packages/core/src/core/errors.ts +++ b/runtime/typescript/packages/core/src/core/errors.ts @@ -17,7 +17,8 @@ export type LoadErrorKind = | "file_not_found" | "invalid_frontmatter" | "invalid_template" - | "missing_required_input"; + | "missing_required_input" + | "untrusted_endpoint"; /** * A typed error raised by the loader/pipeline for negative-path load and diff --git a/runtime/typescript/packages/core/src/core/index.ts b/runtime/typescript/packages/core/src/core/index.ts index f0f59a79a..f96856848 100644 --- a/runtime/typescript/packages/core/src/core/index.ts +++ b/runtime/typescript/packages/core/src/core/index.ts @@ -4,6 +4,12 @@ export * from "./registry.js"; export * from "./connections.js"; export { load, defaultSaveContext, type LoadOptions } from "./loader.js"; export { PromptyLoadError, type LoadErrorKind } from "./errors.js"; +export { + isTrustedEndpoint, + checkConnectionTrust, + BUILTIN_TRUSTED_ENDPOINTS, + TRUSTED_ENDPOINTS_ENV, +} from "./endpoint-trust.js"; export { validateInputs, render, diff --git a/runtime/typescript/packages/core/src/core/loader.ts b/runtime/typescript/packages/core/src/core/loader.ts index 1aa40cfd8..802255883 100644 --- a/runtime/typescript/packages/core/src/core/loader.ts +++ b/runtime/typescript/packages/core/src/core/loader.ts @@ -14,6 +14,7 @@ import matter from "gray-matter"; import { LoadContext, SaveContext } from "../model/context.js"; import { Agent } from "../model/agent.js"; import { PromptyLoadError } from "./errors.js"; +import { checkConnectionTrust } from "./endpoint-trust.js"; export interface LoadOptions { /** @@ -21,6 +22,13 @@ export interface LoadOptions { * The prompt file's directory is always allowed. */ allowedFileRoots?: string[]; + /** + * Additional origins trusted to receive ambient credentials (spec §2.5.1), + * on top of the built-in provider origins and `PROMPTY_TRUSTED_ENDPOINTS`. + * Entries are origins such as `http://localhost:11434` or + * `https://*.contoso.com`; `*` trusts every endpoint. + */ + trustedEndpoints?: string[]; } // --------------------------------------------------------------------------- @@ -132,9 +140,16 @@ function buildAgent( ); } + // Endpoint trust (spec §2.5.1) runs on the RAW connection, before references resolve. + const { agentDir, allowedRoots } = fileRoots(filePath, options); + checkConnectionTrust( + rawConnection(frontmatter, agentDir, allowedRoots), + options.trustedEndpoints, + ); + // 2. Load via Prompty.load() with preProcess for ${protocol:value} expansion const ctx = new LoadContext({ - preProcess: makePreProcess(filePath, options) as ( + preProcess: makePreProcess(agentDir, allowedRoots) as ( data: Record, ) => Record, }); @@ -149,6 +164,64 @@ function buildAgent( return agent; } +function fileRoots( + agentFile: string, + options: LoadOptions, +): { agentDir: string; allowedRoots: string[] } { + const agentDir = realpathSync(dirname(agentFile)); + const allowedRoots = [ + agentDir, + ...(options.allowedFileRoots ?? []).map((root) => + canonicalizeExistingPath(resolve(root)), + ), + ]; + return { agentDir, allowedRoots }; +} + +/** + * Return the unresolved `model.connection`, following whole-value `${file:}` + * references (with the same containment as reference resolution). + */ +function rawConnection( + data: Record, + agentDir: string, + allowedRoots: string[], +): unknown { + const follow = (value: unknown, key: string): unknown => { + if ( + typeof value === "string" && + value.startsWith("${") && + value.endsWith("}") + ) { + const inner = value.slice(2, -1); + const colon = inner.indexOf(":"); + if (colon !== -1 && inner.slice(0, colon).toLowerCase() === "file") { + const reference = inner.slice(colon + 1); + const candidate = isAbsolute(reference) + ? resolve(reference) + : resolve(agentDir, reference); + // Missing files are reported by reference resolution; only containment is enforced here. + if (!existsSync(candidate)) { + if (!isWithinAnyRoot(candidate, allowedRoots)) { + resolveFileReference(agentDir, reference, allowedRoots, key); + } + return undefined; + } + return loadFileContent( + resolveFileReference(agentDir, reference, allowedRoots, key), + ); + } + } + return value; + }; + + const model = follow(data["model"], "model"); + if (typeof model !== "object" || model === null || Array.isArray(model)) { + return undefined; + } + return follow((model as Record)["connection"], "connection"); +} + function rejectExecutableFrontmatter(): never { throw new Error("JavaScript frontmatter is not supported in .prompty files"); } @@ -170,17 +243,9 @@ function rejectExecutableFrontmatter(): never { * provide additional allowed roots via `allowedFileRoots`. */ function makePreProcess( - agentFile: string, - options: LoadOptions, + agentDir: string, + allowedRoots: string[], ): (data: unknown) => unknown { - const agentDir = realpathSync(dirname(agentFile)); - const allowedRoots = [ - agentDir, - ...(options.allowedFileRoots ?? []).map((root) => - canonicalizeExistingPath(resolve(root)), - ), - ]; - return (data: unknown): unknown => { if (typeof data !== "object" || data === null || Array.isArray(data)) { return data; diff --git a/runtime/typescript/packages/core/src/index.ts b/runtime/typescript/packages/core/src/index.ts index a761246a5..cbf804329 100644 --- a/runtime/typescript/packages/core/src/index.ts +++ b/runtime/typescript/packages/core/src/index.ts @@ -58,6 +58,9 @@ export { // Loader load, type LoadOptions, + isTrustedEndpoint, + BUILTIN_TRUSTED_ENDPOINTS, + TRUSTED_ENDPOINTS_ENV, // Pipeline functions validateInputs, diff --git a/runtime/typescript/packages/core/src/model/vector-conformance.ts b/runtime/typescript/packages/core/src/model/vector-conformance.ts index b7307b453..b6a7f2a1f 100644 --- a/runtime/typescript/packages/core/src/model/vector-conformance.ts +++ b/runtime/typescript/packages/core/src/model/vector-conformance.ts @@ -662,4 +662,461 @@ export async function runLoadConformanceConformance( "tools_openapi_load misrouted", ); } + // vector: trusted_endpoint_builtin_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://myres.services.ai.azure.com/api/projects/p"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_builtin_allowed misrouted", + ); + } + // vector: trusted_endpoint_default_port_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://API.OpenAI.com:443/v1"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_default_port_allowed misrouted", + ); + } + // vector: trusted_endpoint_env_endpoint_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://host-chosen.example.com"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_env_endpoint_allowed misrouted", + ); + } + // vector: trusted_endpoint_env_list_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"key","endpoint":"http://localhost:11434/v1"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_env_list_allowed misrouted", + ); + } + // vector: trusted_endpoint_literal_key_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"key","endpoint":"https://anywhere.example.com"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_literal_key_allowed misrouted", + ); + } + // vector: trusted_endpoint_option_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://proxy.example.com:443/openai/v1"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_option_allowed misrouted", + ); + } + // vector: trusted_endpoint_reference_skipped + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"reference","name":"my-connection"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_reference_skipped misrouted", + ); + } + // vector: trusted_endpoint_userinfo_allowed + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + const actual = await seam.load(source); + const expected = JSON.parse( + '{"model":{"connection":{"kind":"foundry","endpoint":"https://user@api.openai.com/v1"}}}', + ); + seamConformanceAssert( + seamConformanceCanonical(JSON.parse(JSON.stringify(actual))) === + seamConformanceCanonical(expected), + "trusted_endpoint_userinfo_allowed misrouted", + ); + } + // vector: untrusted_endpoint_ambient_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_ambient_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_backslash_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_backslash_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_blank_literal_key_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_blank_literal_key_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_env_default_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_env_default_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_env_key_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_env_key_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_file_connection_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_file_connection_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_fragment_before_at_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_fragment_before_at_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_ipv6_literal_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_ipv6_literal_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_key_with_client_secret_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_key_with_client_secret_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_mismatched_credential_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_mismatched_credential_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_no_key_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_no_key_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_oauth_literal_secret_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_oauth_literal_secret_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_percent_encoded_host_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_percent_encoded_host_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_port_mismatch_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_port_mismatch_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_query_before_at_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_query_before_at_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_scheme_mismatch_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_scheme_mismatch_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_suffix_lookalike_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_suffix_lookalike_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_suffix_spoof_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_suffix_spoof_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_trailing_dot_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_trailing_dot_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_unicode_homoglyph_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_unicode_homoglyph_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_userinfo_lookalike_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_userinfo_lookalike_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_wildcard_apex_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_wildcard_apex_rejected: expected an error", + ); + } + // vector: untrusted_endpoint_wildcard_boundary_rejected + { + const sourceJson = "null"; + const source = JSON.parse(sourceJson) as unknown; + let caught: unknown; + try { + await seam.load(source); + } catch (error) { + caught = error; + } + seamConformanceAssert( + caught !== undefined, + "untrusted_endpoint_wildcard_boundary_rejected: expected an error", + ); + } } diff --git a/runtime/typescript/packages/core/tests/endpoint-trust.test.ts b/runtime/typescript/packages/core/tests/endpoint-trust.test.ts new file mode 100644 index 000000000..de5773485 --- /dev/null +++ b/runtime/typescript/packages/core/tests/endpoint-trust.test.ts @@ -0,0 +1,241 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { load } from "../src/core/loader.js"; +import { isTrustedEndpoint } from "../src/core/endpoint-trust.js"; +import { PromptyLoadError } from "../src/core/errors.js"; + +// Endpoint trust (spec §2.5.1). Mirrors runtime/python/prompty/tests/test_endpoint_trust.py. + +const ENV_KEYS = [ + "PROMPTY_TRUSTED_ENDPOINTS", + "SOME_KEY", + "SECRET", + "EP", + "UNSET_EP", +]; +let savedEnv: Record; +let dir: string; + +beforeEach(() => { + savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); + for (const k of ENV_KEYS) delete process.env[k]; + dir = mkdtempSync(join(tmpdir(), "prompty-trust-")); +}); + +afterEach(() => { + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + rmSync(dir, { recursive: true, force: true }); +}); + +function write(connection: string, base: string = dir): string { + const p = join(base, "t.prompty"); + writeFileSync( + p, + `---\nname: t\nmodel:\n id: gpt-4\n connection:\n${connection}\n---\nHello\n`, + "utf-8", + ); + return p; +} + +function loadError(fn: () => unknown): PromptyLoadError { + try { + fn(); + } catch (err) { + expect(err).toBeInstanceOf(PromptyLoadError); + return err as PromptyLoadError; + } + throw new Error("expected load to throw"); +} + +describe("isTrustedEndpoint", () => { + it.each([ + "https://api.openai.com/v1", + "https://API.OpenAI.com:443", + "https://myres.openai.azure.com/", + "https://a.b.services.ai.azure.com/api/projects/p", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + ])("trusts built-in %s", (url) => { + expect(isTrustedEndpoint(url)).toBe(true); + }); + + it.each([ + "http://api.openai.com", // scheme mismatch + "https://api.openai.com:8443", // port mismatch + "https://openai.azure.com", // wildcard never matches apex + "https://api.openai.com.evil.com", + "https://evilopenai.azure.com", + "https://api.openai.com@evil.com", // userinfo ignored; host is evil.com + "http://localhost:11434", + "not a url", + "ftp://api.openai.com", + "https:api.openai.com", // not an absolute URL + "", + ])("rejects %s", (url) => { + expect(isTrustedEndpoint(url)).toBe(false); + }); + + it("honors host option entries and wildcards", () => { + expect( + isTrustedEndpoint("http://localhost:11434/v1", [ + "http://LOCALHOST:11434", + ]), + ).toBe(true); + expect( + isTrustedEndpoint("https://x.contoso.com", ["https://*.contoso.com"]), + ).toBe(true); + expect( + isTrustedEndpoint("https://contoso.com", ["https://*.contoso.com"]), + ).toBe(false); + }); + + it("honors PROMPTY_TRUSTED_ENDPOINTS and '*'", () => { + process.env.PROMPTY_TRUSTED_ENDPOINTS = + " https://a.example.com , http://localhost:8080"; + expect(isTrustedEndpoint("http://localhost:8080/x")).toBe(true); + expect(isTrustedEndpoint("http://localhost:8081")).toBe(false); + process.env.PROMPTY_TRUSTED_ENDPOINTS = "*"; + expect(isTrustedEndpoint("http://anything.example")).toBe(true); + }); +}); + +describe("loader enforcement", () => { + it("rejects an ambient credential with a literal endpoint", () => { + const p = write(" kind: foundry\n endpoint: http://127.0.0.1:9999"); + const err = loadError(() => load(p)); + expect(err.kind).toBe("untrusted_endpoint"); + expect(err.message).toContain("127.0.0.1"); + }); + + it("rejects an env-supplied key with a literal endpoint", () => { + process.env.SOME_KEY = "secret"; + const p = write( + " kind: key\n endpoint: https://evil.example.com\n apiKey: ${env:SOME_KEY}", + ); + expect(() => load(p)).toThrow(/not trusted/); + }); + + it("checks oauth tokenUrl", () => { + process.env.SECRET = "s"; + const p = write( + " kind: oauth\n endpoint: https://myres.openai.azure.com\n clientId: c\n" + + " clientSecret: ${env:SECRET}\n tokenUrl: https://evil.example.com/token", + ); + expect(() => load(p)).toThrow(/tokenUrl/); + }); + + it("ignores credential fields the kind does not consume", () => { + let p = write( + " kind: foundry\n endpoint: https://evil.example.com\n apiKey: junk", + ); + expect(() => load(p)).toThrow(/not trusted/); + p = write( + " kind: key\n endpoint: https://evil.example.com\n clientSecret: junk", + ); + expect(() => load(p)).toThrow(/not trusted/); + }); + + it("allows a literal key with any endpoint", () => { + const agent = load( + write( + " kind: key\n endpoint: http://127.0.0.1:9999\n apiKey: mine", + ), + ); + expect((agent.model?.connection as any).endpoint).toBe( + "http://127.0.0.1:9999", + ); + }); + + it("allows a host-controlled ${env:} endpoint", () => { + process.env.EP = "http://127.0.0.1:9999"; + const agent = load(write(" kind: foundry\n endpoint: ${env:EP}")); + expect((agent.model?.connection as any).endpoint).toBe( + "http://127.0.0.1:9999", + ); + }); + + it("rejects an ${env:} default used because the var is unset", () => { + const p = write( + " kind: foundry\n endpoint: ${env:UNSET_EP:http://127.0.0.1:9999}", + ); + expect(() => load(p)).toThrow(/not trusted/); + }); + + it("allows origins passed via trustedEndpoints", () => { + const p = write( + " kind: foundry\n endpoint: http://127.0.0.1:9999/api", + ); + const agent = load(p, { trustedEndpoints: ["http://127.0.0.1:9999"] }); + expect((agent.model?.connection as any).endpoint).toBe( + "http://127.0.0.1:9999/api", + ); + }); + + it("skips kind: reference", () => { + const agent = load( + write( + " kind: reference\n name: my-conn\n target: https://evil.example.com", + ), + ); + expect((agent.model?.connection as any).kind).toBe("reference"); + }); + + it("follows a whole-value ${file:} model reference", () => { + writeFileSync( + join(dir, "model.json"), + JSON.stringify({ + id: "gpt-4", + connection: { kind: "foundry", endpoint: "https://evil.example.com" }, + }), + "utf-8", + ); + const p = join(dir, "t.prompty"); + writeFileSync(p, "---\nname: t\nmodel: ${file:model.json}\n---\nHi\n"); + expect(() => load(p)).toThrow(/not trusted/); + }); + + it("follows a whole-value ${file:} connection reference", () => { + writeFileSync( + join(dir, "conn.json"), + JSON.stringify({ kind: "foundry", endpoint: "https://evil.example.com" }), + "utf-8", + ); + const p = join(dir, "t.prompty"); + writeFileSync( + p, + "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:conn.json}\n---\nHi\n", + ); + const err = loadError(() => load(p)); + expect(err.kind).toBe("untrusted_endpoint"); + }); + + it("reports traversal ${file:} as file_reference before trust", () => { + const promptDir = join(dir, "prompts"); + mkdirSync(promptDir); + writeFileSync( + join(dir, "conn.json"), + JSON.stringify({ kind: "foundry", endpoint: "https://evil.example.com" }), + "utf-8", + ); + const p = join(promptDir, "t.prompty"); + writeFileSync( + p, + "---\nname: t\nmodel:\n id: gpt-4\n connection: ${file:../conn.json}\n---\nHi\n", + ); + expect(loadError(() => load(p)).kind).toBe("file_reference"); + }); + + it("does not let frontmatter grant itself trust", () => { + const p = join(dir, "t.prompty"); + writeFileSync( + p, + "---\nname: t\nmetadata:\n trustedEndpoints: ['*']\n PROMPTY_TRUSTED_ENDPOINTS: '*'\n" + + "model:\n id: gpt-4\n connection:\n kind: foundry\n endpoint: https://evil.example.com\n---\nHi\n", + ); + expect(loadError(() => load(p)).kind).toBe("untrusted_endpoint"); + }); +}); diff --git a/runtime/typescript/packages/core/tests/model/vector-adapters.ts b/runtime/typescript/packages/core/tests/model/vector-adapters.ts index 176eacf0c..a271313f6 100644 --- a/runtime/typescript/packages/core/tests/model/vector-adapters.ts +++ b/runtime/typescript/packages/core/tests/model/vector-adapters.ts @@ -86,7 +86,7 @@ import { type TurnToolCall, type TurnToolResult, } from "../../src/index.js"; -import { defaultSaveContext } from "../../src/core/loader.js"; +import { defaultSaveContext, type LoadOptions } from "../../src/core/loader.js"; import { TurnOptions } from "../../src/model/index.js"; import { enrich, mapModel } from "../../src/model/discovery.js"; import { ModelInfo } from "../../src/model/contracts/models/model-info.js"; @@ -375,6 +375,16 @@ function loadInvoke(input: any, _context: AdapterContext): unknown { oldEnv[k] = process.env[k]; process.env[k] = v; } + // Endpoint-trust vectors must not see an ambient PROMPTY_TRUSTED_ENDPOINTS. + if (!("PROMPTY_TRUSTED_ENDPOINTS" in envVars)) { + if (!("PROMPTY_TRUSTED_ENDPOINTS" in oldEnv)) { + oldEnv.PROMPTY_TRUSTED_ENDPOINTS = process.env.PROMPTY_TRUSTED_ENDPOINTS; + } + delete process.env.PROMPTY_TRUSTED_ENDPOINTS; + } + const loadOptions: LoadOptions = { + trustedEndpoints: input.trustedEndpoints, + }; try { // --- input-validation vectors --- @@ -390,18 +400,20 @@ function loadInvoke(input: any, _context: AdapterContext): unknown { const base = mkdtempSync(join(tmpdir(), "prompty-vec-")); try { if ("fixture" in input) { - return saveCanonical(load(join(SPEC_FIXTURES, input.fixture))); + return saveCanonical( + load(join(SPEC_FIXTURES, input.fixture), loadOptions), + ); } if ("frontmatter_raw" in input) { const p = join(base, "vector.prompty"); writeFileSync(p, input.frontmatter_raw, "utf8"); - return saveCanonical(load(p)); + return saveCanonical(load(p, loadOptions)); } const sub = input.agent_subdir ? join(base, input.agent_subdir) : base; mkdirSync(sub, { recursive: true }); const p = join(sub, "vector.prompty"); writePrompty(p, input.frontmatter, input.files); - return saveCanonical(load(p)); + return saveCanonical(load(p, loadOptions)); } finally { rmSync(base, { recursive: true, force: true }); } diff --git a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts index af3529740..ff862a14f 100644 --- a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts +++ b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts @@ -167,7 +167,7 @@ describe("callable vector conformance", () => { }); it("LoadConformance.load:connection_types_load", async () => { const vector = JSON.parse( - '{"name":"connection_types_load","description":"All connection kinds (key, reference, remote, anonymous) load correctly","stage":"load","input":{"frontmatter":{"name":"connection-test","model":{"id":"gpt-4","connection":{"kind":"anonymous","endpoint":"https://localhost:8080"}}}},"expected":{"model":{"id":"gpt-4","connection":{"kind":"anonymous","endpoint":"https://localhost:8080"}}},"operation":"load"}', + '{"name":"connection_types_load","description":"All connection kinds (key, reference, remote, anonymous) load correctly","stage":"load","input":{"frontmatter":{"name":"connection-test","model":{"id":"gpt-4","connection":{"kind":"anonymous","endpoint":"https://localhost:8080"}}},"trustedEndpoints":["https://localhost:8080"]},"expected":{"model":{"id":"gpt-4","connection":{"kind":"anonymous","endpoint":"https://localhost:8080"}}},"operation":"load"}', ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); @@ -185,7 +185,7 @@ describe("callable vector conformance", () => { }); it("LoadConformance.load:env_default", async () => { const vector = JSON.parse( - '{"name":"env_default","description":"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set","stage":"load","input":{"frontmatter":{"name":"env-default-test","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"${env:MISSING_VAR:fallback_value}"}}},"env":{}},"expected":{"model":{"connection":{"kind":"key","endpoint":"fallback_value"}}},"operation":"load"}', + '{"name":"env_default","description":"${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set","stage":"load","input":{"frontmatter":{"name":"env-default-test","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"${env:MISSING_VAR:fallback_value}","apiKey":"literal-key"}}},"env":{}},"expected":{"model":{"connection":{"kind":"key","endpoint":"fallback_value"}}},"operation":"load"}', ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); @@ -327,6 +327,192 @@ describe("callable vector conformance", () => { ) as Record; await runVector("LoadConformance", "load", vector, false, seam); }); + it("LoadConformance.load:trusted_endpoint_builtin_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_builtin_allowed","description":"Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration","stage":"load","input":{"frontmatter":{"name":"trusted-builtin","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://myres.services.ai.azure.com/api/projects/p"}}}},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://myres.services.ai.azure.com/api/projects/p"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_default_port_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_default_port_allowed","description":"An explicit default port normalizes to the built-in origin","stage":"load","input":{"frontmatter":{"name":"trusted-default-port-allowed","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://API.OpenAI.com:443/v1"}}}},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://API.OpenAI.com:443/v1"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_env_endpoint_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_env_endpoint_allowed","description":"An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials","stage":"load","input":{"frontmatter":{"name":"trusted-env-endpoint","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"${env:VECTOR_FOUNDRY_ENDPOINT}"}}},"env":{"VECTOR_FOUNDRY_ENDPOINT":"https://host-chosen.example.com"}},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://host-chosen.example.com"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_env_list_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_env_list_allowed","description":"PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint","stage":"load","input":{"frontmatter":{"name":"trusted-env-list","model":{"id":"llama3","connection":{"kind":"key","endpoint":"http://localhost:11434/v1"}}},"env":{"PROMPTY_TRUSTED_ENDPOINTS":"https://other.example.com, http://localhost:11434"}},"expected":{"model":{"connection":{"kind":"key","endpoint":"http://localhost:11434/v1"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_literal_key_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_literal_key_allowed","description":"A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk","stage":"load","input":{"frontmatter":{"name":"trusted-literal-key","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"https://anywhere.example.com","apiKey":"file-supplied-key"}}}},"expected":{"model":{"connection":{"kind":"key","endpoint":"https://anywhere.example.com"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_option_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_option_allowed","description":"A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored","stage":"load","input":{"frontmatter":{"name":"trusted-option","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://proxy.example.com:443/openai/v1"}}},"trustedEndpoints":["HTTPS://PROXY.Example.com"]},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://proxy.example.com:443/openai/v1"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_reference_skipped", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_reference_skipped","description":"kind: reference connections carry no endpoint and are not subject to the trust check","stage":"load","input":{"frontmatter":{"name":"trusted-reference","model":{"id":"gpt-4","connection":{"kind":"reference","name":"my-connection"}}}},"expected":{"model":{"connection":{"kind":"reference","name":"my-connection"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:trusted_endpoint_userinfo_allowed", async () => { + const vector = JSON.parse( + '{"name":"trusted_endpoint_userinfo_allowed","description":"Userinfo is ignored; the destination host api.openai.com is trusted","stage":"load","input":{"frontmatter":{"name":"trusted-userinfo-allowed","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://user@api.openai.com/v1"}}}},"expected":{"model":{"connection":{"kind":"foundry","endpoint":"https://user@api.openai.com/v1"}}},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_ambient_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_ambient_rejected","description":"A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)","stage":"load","input":{"frontmatter":{"name":"untrusted-ambient","model":{"id":"gpt-4","provider":"foundry","connection":{"kind":"foundry","endpoint":"https://attacker.example.com/api/projects/p"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_backslash_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_backslash_rejected","description":"A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted","stage":"load","input":{"frontmatter":{"name":"untrusted-backslash","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://evil.example.com\\\\@api.openai.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_blank_literal_key_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_blank_literal_key_rejected","description":"a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials","stage":"load","input":{"frontmatter":{"name":"untrusted-blank-key","model":{"id":"gpt-4","provider":"foundry","connection":{"kind":"key","endpoint":"https://attacker.example.com","apiKey":" "}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_env_default_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_env_default_rejected","description":"An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-env-default","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"${env:NONEXISTENT:https://attacker.example.com}"}}},"env":{}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_env_key_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_env_key_rejected","description":"A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose","stage":"load","input":{"frontmatter":{"name":"untrusted-env-key","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"https://attacker.example.com","apiKey":"${env:VECTOR_HOST_KEY}"}}},"env":{"VECTOR_HOST_KEY":"host-secret"}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_file_connection_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_file_connection_rejected","description":"A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-file-conn","model":{"id":"gpt-4","connection":"${file:conn.json}"}},"files":{"conn.json":{"kind":"foundry","endpoint":"https://attacker.example.com"}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_fragment_before_at_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_fragment_before_at_rejected","description":"A \'#\' ends the authority, so the host is evil.example.com","stage":"load","input":{"frontmatter":{"name":"untrusted-fragment-before-at","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://evil.example.com#@api.openai.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_ipv6_literal_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_ipv6_literal_rejected","description":"An IPv6 literal is not a trusted origin","stage":"load","input":{"frontmatter":{"name":"untrusted-ipv6-literal","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://[::1]:8080"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_key_with_client_secret_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_key_with_client_secret_rejected","description":"A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-key-client-secret","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"https://attacker.example.com","clientSecret":"junk"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_mismatched_credential_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_mismatched_credential_rejected","description":"A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied","stage":"load","input":{"frontmatter":{"name":"untrusted-mismatched-credential","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://attacker.example.com","apiKey":"junk"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_no_key_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_no_key_rejected","description":"A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected","stage":"load","input":{"frontmatter":{"name":"untrusted-no-key","model":{"id":"gpt-4","connection":{"kind":"key","endpoint":"http://127.0.0.1:8080"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_oauth_literal_secret_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_oauth_literal_secret_rejected","description":"kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials","stage":"load","input":{"frontmatter":{"name":"untrusted-oauth-literal","model":{"id":"gpt-4","provider":"anthropic","connection":{"kind":"oauth","endpoint":"https://attacker.example.com","clientId":"decoy","clientSecret":"decoy","tokenUrl":"https://login.microsoftonline.com/t/oauth2/v2.0/token"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_percent_encoded_host_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_percent_encoded_host_rejected","description":"Hosts are compared without percent-decoding, so an encoded host never matches","stage":"load","input":{"frontmatter":{"name":"untrusted-percent-encoded-host","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com%2eevil.example.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_port_mismatch_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_port_mismatch_rejected","description":"A non-default port is a different origin","stage":"load","input":{"frontmatter":{"name":"untrusted-port-mismatch","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com:8443"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_query_before_at_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_query_before_at_rejected","description":"A \'?\' ends the authority, so the host is evil.example.com","stage":"load","input":{"frontmatter":{"name":"untrusted-query-before-at","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://evil.example.com?@api.openai.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_scheme_mismatch_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_scheme_mismatch_rejected","description":"Built-in origins are https; http on the same host is a different origin","stage":"load","input":{"frontmatter":{"name":"untrusted-scheme-mismatch","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"http://api.openai.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_suffix_lookalike_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_suffix_lookalike_rejected","description":"A trusted host as a label prefix of another domain is not trusted","stage":"load","input":{"frontmatter":{"name":"untrusted-suffix-lookalike","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com.evil.example.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_suffix_spoof_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_suffix_spoof_rejected","description":"Origin matching is exact: a host that merely starts with a trusted host is not trusted","stage":"load","input":{"frontmatter":{"name":"untrusted-spoof","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://good.example.com.attacker.example.com"}}},"trustedEndpoints":["https://good.example.com"]},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_trailing_dot_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_trailing_dot_rejected","description":"Hosts are compared exactly; a trailing-dot host does not match","stage":"load","input":{"frontmatter":{"name":"untrusted-trailing-dot","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com."}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_unicode_homoglyph_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_unicode_homoglyph_rejected","description":"A non-ASCII lookalike host does not match","stage":"load","input":{"frontmatter":{"name":"untrusted-unicode-homoglyph","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://аpi.openai.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_userinfo_lookalike_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_userinfo_lookalike_rejected","description":"A trusted host in userinfo is not the destination host","stage":"load","input":{"frontmatter":{"name":"untrusted-userinfo-lookalike","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://api.openai.com@evil.example.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_wildcard_apex_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_wildcard_apex_rejected","description":"A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme","stage":"load","input":{"frontmatter":{"name":"untrusted-apex","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"http://myres.openai.azure.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); + it("LoadConformance.load:untrusted_endpoint_wildcard_boundary_rejected", async () => { + const vector = JSON.parse( + '{"name":"untrusted_endpoint_wildcard_boundary_rejected","description":"*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com","stage":"load","input":{"frontmatter":{"name":"untrusted-wildcard-boundary","model":{"id":"gpt-4","connection":{"kind":"foundry","endpoint":"https://evilservices.ai.azure.com"}}}},"expectedError":{"kind":"untrusted_endpoint"},"operation":"load"}', + ) as Record; + await runVector("LoadConformance", "load", vector, false, seam); + }); it("MemoryConformance.operate:clear_by_category_and_all", async () => { const vector = JSON.parse( '{"name":"clear_by_category_and_all","description":"Clear removes only the requested category, or all memories when no category is supplied.","stage":"memory","input":{"operation":"clear","store":{"entries":[{"content":"fact","category":"core"},{"content":"summary","category":"archival"},{"content":"another summary","category":"archival"},{"content":"insight","category":"insight"}]},"category":"archival"},"expected":{"removed":2,"store":{"entries":[{"content":"fact","category":"core"},{"content":"insight","category":"insight"}]}},"operation":"operate"}', diff --git a/schema/model/conformance/vectors/load.tsp b/schema/model/conformance/vectors/load.tsp index 968c2c1d3..1fb6cec47 100644 --- a/schema/model/conformance/vectors/load.tsp +++ b/schema/model/conformance/vectors/load.tsp @@ -146,7 +146,8 @@ const LoadVectors = """ "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "\${env:MISSING_VAR:fallback_value}" + "endpoint": "\${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }, @@ -269,6 +270,692 @@ const LoadVectors = """ "kind": "file_reference" } }, + { + "name": "untrusted_endpoint_ambient_rejected", + "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ambient", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com/api/projects/p" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_env_key_rejected", + "description": "A literal endpoint with an \${env:} apiKey is rejected: the file would route a host secret to a destination it chose", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": "\${env:VECTOR_HOST_KEY}" + } + } + }, + "env": { + "VECTOR_HOST_KEY": "host-secret" + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_no_key_rejected", + "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-no-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "http://127.0.0.1:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_mismatched_credential_rejected", + "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-mismatched-credential", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com", + "apiKey": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_key_with_client_secret_rejected", + "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-key-client-secret", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "clientSecret": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_blank_literal_key_rejected", + "description": "a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-blank-key", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": " " + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_oauth_literal_secret_rejected", + "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-oauth-literal", + "model": { + "id": "gpt-4", + "provider": "anthropic", + "connection": { + "kind": "oauth", + "endpoint": "https://attacker.example.com", + "clientId": "decoy", + "clientSecret": "decoy", + "tokenUrl": "https://login.microsoftonline.com/t/oauth2/v2.0/token" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_backslash_rejected", + "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-backslash", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com\\\\@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_fragment_before_at_rejected", + "description": "A '#' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-fragment-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com#@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_query_before_at_rejected", + "description": "A '?' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-query-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com?@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_userinfo_lookalike_rejected", + "description": "A trusted host in userinfo is not the destination host", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-userinfo-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com@evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_suffix_lookalike_rejected", + "description": "A trusted host as a label prefix of another domain is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-suffix-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com.evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_wildcard_boundary_rejected", + "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-wildcard-boundary", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evilservices.ai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_scheme_mismatch_rejected", + "description": "Built-in origins are https; http on the same host is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-scheme-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_port_mismatch_rejected", + "description": "A non-default port is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-port-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com:8443" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_ipv6_literal_rejected", + "description": "An IPv6 literal is not a trusted origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ipv6-literal", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://[::1]:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_percent_encoded_host_rejected", + "description": "Hosts are compared without percent-decoding, so an encoded host never matches", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-percent-encoded-host", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com%2eevil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_trailing_dot_rejected", + "description": "Hosts are compared exactly; a trailing-dot host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-trailing-dot", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com." + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_unicode_homoglyph_rejected", + "description": "A non-ASCII lookalike host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-unicode-homoglyph", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://аpi.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "trusted_endpoint_userinfo_allowed", + "description": "Userinfo is ignored; the destination host api.openai.com is trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-userinfo-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + { + "name": "trusted_endpoint_default_port_allowed", + "description": "An explicit default port normalizes to the built-in origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-default-port-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + { + "name": "untrusted_endpoint_env_default_rejected", + "description": "An \${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-default", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "\${env:NONEXISTENT:https://attacker.example.com}" + } + } + }, + "env": {} + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_file_connection_rejected", + "description": "A connection loaded via \${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-file-conn", + "model": { + "id": "gpt-4", + "connection": "\${file:conn.json}" + } + }, + "files": { + "conn.json": { + "kind": "foundry", + "endpoint": "https://attacker.example.com" + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_suffix_spoof_rejected", + "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-spoof", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://good.example.com.attacker.example.com" + } + } + }, + "trustedEndpoints": ["https://good.example.com"] + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "untrusted_endpoint_wildcard_apex_rejected", + "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-apex", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://myres.openai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + } + }, + { + "name": "trusted_endpoint_env_endpoint_allowed", + "description": "An \${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-endpoint", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "\${env:VECTOR_FOUNDRY_ENDPOINT}" + } + } + }, + "env": { + "VECTOR_FOUNDRY_ENDPOINT": "https://host-chosen.example.com" + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://host-chosen.example.com" + } + } + } + }, + { + "name": "trusted_endpoint_literal_key_allowed", + "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-literal-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com", + "apiKey": "file-supplied-key" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com" + } + } + } + }, + { + "name": "trusted_endpoint_option_allowed", + "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-option", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "trustedEndpoints": ["HTTPS://PROXY.Example.com"] + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + } + }, + { + "name": "trusted_endpoint_env_list_allowed", + "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-list", + "model": { + "id": "llama3", + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "env": { + "PROMPTY_TRUSTED_ENDPOINTS": "https://other.example.com, http://localhost:11434" + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + } + }, + { + "name": "trusted_endpoint_builtin_allowed", + "description": "Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-builtin", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + { + "name": "trusted_endpoint_reference_skipped", + "description": "kind: reference connections carry no endpoint and are not subject to the trust check", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-reference", + "model": { + "id": "gpt-4", + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, { "name": "tools_function_load", "description": "Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings", @@ -676,7 +1363,8 @@ const LoadVectors = """ "endpoint": "https://localhost:8080" } } - } + }, + "trustedEndpoints": ["https://localhost:8080"] }, "expected": { "model": { diff --git a/schema/tsp-output/.typra-generated/vectors.json b/schema/tsp-output/.typra-generated/vectors.json index 23feeae50..7dfbaa19a 100644 --- a/schema/tsp-output/.typra-generated/vectors.json +++ b/schema/tsp-output/.typra-generated/vectors.json @@ -841,7 +841,10 @@ "endpoint": "https://localhost:8080" } } - } + }, + "trustedEndpoints": [ + "https://localhost:8080" + ] }, "expected": { "model": { @@ -950,7 +953,8 @@ "id": "gpt-4", "connection": { "kind": "key", - "endpoint": "${env:MISSING_VAR:fallback_value}" + "endpoint": "${env:MISSING_VAR:fallback_value}", + "apiKey": "literal-key" } } }, @@ -1853,6 +1857,1068 @@ "operation": "load" } }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_builtin_allowed", + "description": "Built-in first-party provider origins (here https://*.services.ai.azure.com) are trusted without host configuration", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-builtin", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://myres.services.ai.azure.com/api/projects/p" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_default_port_allowed", + "description": "An explicit default port normalizes to the built-in origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-default-port-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://API.OpenAI.com:443/v1" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_env_endpoint_allowed", + "description": "An ${env:VAR} endpoint whose VAR is set is host-controlled and allowed with ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-endpoint", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:VECTOR_FOUNDRY_ENDPOINT}" + } + } + }, + "env": { + "VECTOR_FOUNDRY_ENDPOINT": "https://host-chosen.example.com" + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://host-chosen.example.com" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_env_list_allowed", + "description": "PROMPTY_TRUSTED_ENDPOINTS (comma-separated, whitespace-trimmed) admits a local endpoint", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-env-list", + "model": { + "id": "llama3", + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "env": { + "PROMPTY_TRUSTED_ENDPOINTS": "https://other.example.com, http://localhost:11434" + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "http://localhost:11434/v1" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_literal_key_allowed", + "description": "A file-supplied literal apiKey may go to any endpoint: no ambient credential is at risk", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-literal-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com", + "apiKey": "file-supplied-key" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "key", + "endpoint": "https://anywhere.example.com" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_option_allowed", + "description": "A host-provided trustedEndpoints origin admits the endpoint; scheme/host case and default port are normalized and the path is ignored", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-option", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "trustedEndpoints": [ + "HTTPS://PROXY.Example.com" + ] + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://proxy.example.com:443/openai/v1" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_reference_skipped", + "description": "kind: reference connections carry no endpoint and are not subject to the trust check", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-reference", + "model": { + "id": "gpt-4", + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "reference", + "name": "my-connection" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "trusted_endpoint_userinfo_allowed", + "description": "Userinfo is ignored; the destination host api.openai.com is trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "trusted-userinfo-allowed", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + } + }, + "expected": { + "model": { + "connection": { + "kind": "foundry", + "endpoint": "https://user@api.openai.com/v1" + } + } + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_ambient_rejected", + "description": "A literal endpoint with no file-supplied credential (ambient identity) is rejected unless its origin is trusted (spec §2.5.1)", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ambient", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com/api/projects/p" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_backslash_rejected", + "description": "A backslash makes parsers disagree on the host (clients connect to evil.com), so the destination is untrusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-backslash", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com\\@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_blank_literal_key_rejected", + "description": "a whitespace-only literal apiKey is ambient: executors treat a blank key as absent and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-blank-key", + "model": { + "id": "gpt-4", + "provider": "foundry", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": " " + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_env_default_rejected", + "description": "An ${env:VAR:default} endpoint whose VAR is unset uses a file-chosen default, so it is file-controlled and rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-default", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "${env:NONEXISTENT:https://attacker.example.com}" + } + } + }, + "env": {} + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_env_key_rejected", + "description": "A literal endpoint with an ${env:} apiKey is rejected: the file would route a host secret to a destination it chose", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-env-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "apiKey": "${env:VECTOR_HOST_KEY}" + } + } + }, + "env": { + "VECTOR_HOST_KEY": "host-secret" + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_file_connection_rejected", + "description": "A connection loaded via ${file:} is file-controlled; an ambient-credential endpoint inside it is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-file-conn", + "model": { + "id": "gpt-4", + "connection": "${file:conn.json}" + } + }, + "files": { + "conn.json": { + "kind": "foundry", + "endpoint": "https://attacker.example.com" + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_fragment_before_at_rejected", + "description": "A '#' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-fragment-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com#@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_ipv6_literal_rejected", + "description": "An IPv6 literal is not a trusted origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-ipv6-literal", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://[::1]:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_key_with_client_secret_rejected", + "description": "A literal clientSecret on kind key is ignored, so the apiKey stays ambient and the endpoint is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-key-client-secret", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "https://attacker.example.com", + "clientSecret": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_mismatched_credential_rejected", + "description": "A literal apiKey on a kind that does not consume it (foundry) does not make the credential file-supplied", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-mismatched-credential", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://attacker.example.com", + "apiKey": "junk" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_no_key_rejected", + "description": "A key connection with a literal endpoint and no apiKey (executor env-key fallback) is rejected", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-no-key", + "model": { + "id": "gpt-4", + "connection": { + "kind": "key", + "endpoint": "http://127.0.0.1:8080" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_oauth_literal_secret_rejected", + "description": "kind oauth is ambient even with a literal clientSecret: executors do not consume it and fall back to ambient credentials", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-oauth-literal", + "model": { + "id": "gpt-4", + "provider": "anthropic", + "connection": { + "kind": "oauth", + "endpoint": "https://attacker.example.com", + "clientId": "decoy", + "clientSecret": "decoy", + "tokenUrl": "https://login.microsoftonline.com/t/oauth2/v2.0/token" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_percent_encoded_host_rejected", + "description": "Hosts are compared without percent-decoding, so an encoded host never matches", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-percent-encoded-host", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com%2eevil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_port_mismatch_rejected", + "description": "A non-default port is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-port-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com:8443" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_query_before_at_rejected", + "description": "A '?' ends the authority, so the host is evil.example.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-query-before-at", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evil.example.com?@api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_scheme_mismatch_rejected", + "description": "Built-in origins are https; http on the same host is a different origin", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-scheme-mismatch", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://api.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_suffix_lookalike_rejected", + "description": "A trusted host as a label prefix of another domain is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-suffix-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com.evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_suffix_spoof_rejected", + "description": "Origin matching is exact: a host that merely starts with a trusted host is not trusted", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-spoof", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://good.example.com.attacker.example.com" + } + } + }, + "trustedEndpoints": [ + "https://good.example.com" + ] + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_trailing_dot_rejected", + "description": "Hosts are compared exactly; a trailing-dot host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-trailing-dot", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com." + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_unicode_homoglyph_rejected", + "description": "A non-ASCII lookalike host does not match", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-unicode-homoglyph", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://аpi.openai.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_userinfo_lookalike_rejected", + "description": "A trusted host in userinfo is not the destination host", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-userinfo-lookalike", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://api.openai.com@evil.example.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_wildcard_apex_rejected", + "description": "A wildcard origin (https://*.openai.azure.com) matches strict subdomains only, never the apex or another scheme", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-apex", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "http://myres.openai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, + { + "contract": "LoadConformance", + "namespace": "Prompty.Conformance.Seams", + "group": "", + "operation": "load", + "params": { + "source": "unknown" + }, + "returns": "Agent", + "sync": false, + "vector": { + "name": "untrusted_endpoint_wildcard_boundary_rejected", + "description": "*.services.ai.azure.com matches strict subdomains only, not evilservices.ai.azure.com", + "stage": "load", + "input": { + "frontmatter": { + "name": "untrusted-wildcard-boundary", + "model": { + "id": "gpt-4", + "connection": { + "kind": "foundry", + "endpoint": "https://evilservices.ai.azure.com" + } + } + } + }, + "expectedError": { + "kind": "untrusted_endpoint" + }, + "operation": "load" + } + }, { "contract": "MemoryConformance", "namespace": "Prompty.Conformance.Seams", diff --git a/spec/spec.md b/spec/spec.md index 2bbffa218..cf25a8965 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -285,6 +285,58 @@ Connection types are discriminated by the `kind` field: | `foundry` | `endpoint` | Microsoft Foundry connection | | `oauth` | `endpoint`, `authenticationMode` | OAuth-based authentication | +#### §2.5.1 Endpoint Trust + +A `.prompty` file is untrusted input. Executors MAY attach **ambient credentials** +(credentials the file did not supply — e.g. `DefaultAzureCredential` tokens, or +`AZURE_OPENAI_API_KEY` / `OPENAI_API_KEY` env fallbacks) to `model.connection.endpoint`. +A file MUST NOT be able to direct ambient credentials to a destination it chose. + +Loaders MUST enforce this at load time (§4.2 step 5) on `model.connection`: + +1. Skip the check when `connection` is absent or `kind` is `reference`. +2. The **credential is file-supplied** only when `kind` is `key` and `apiKey` is a + literal string that is not blank (contains a non-whitespace character) and is not a + `${...}` reference. A blank `apiKey` is ambient because executors treat it as absent + and fall back to ambient credentials. Every other case is ambient, + including `${env:}` or `${file:}` keys, any other `kind` (e.g. `foundry`, `oauth`, + `anonymous`, custom), and an `apiKey` on any other `kind`. (`oauth` is ambient because + executors do not consume `clientSecret` and fall back to ambient credentials.) +3. Each **destination** (`endpoint`, and `tokenUrl` when present) is **host-controlled** + when it is absent, or is `${env:VAR}` / `${env:VAR:default}` and `VAR` is set. + A literal value, a `${file:}` value (always untrusted), or an `${env:}` default used + because `VAR` is unset is **file-controlled**. +4. If the credential is ambient and any file-controlled destination is not **trusted**, + raise `untrusted_endpoint`. + +When `model` or `model.connection` is itself a `${file:...}` reference, loaders MUST +resolve it (with §2.11 containment) and apply the same classification to its contents. + +**Trusted origins** are the union of: + +| Source | Example | +| --------------------------------------------- | ----------------------------------------- | +| Built-in first-party provider origins | see below | +| Host load option `trustedEndpoints` | `["https://my-proxy.contoso.com"]` | +| Env var `PROMPTY_TRUSTED_ENDPOINTS` (comma-separated) | `http://localhost:11434,https://*.contoso.com` | + +Built-in: `https://api.openai.com`, `https://api.anthropic.com`, +`https://*.openai.azure.com`, `https://*.services.ai.azure.com`, +`https://*.cognitiveservices.azure.com`, `https://login.microsoftonline.com`. + +**Matching**: compare origins only (scheme + host + port). Scheme and host are +case-insensitive; default ports (`http`:80, `https`:443) are normalized; path, query, +and userinfo are ignored. The authority ends at the first `/`, `?`, or `#`. Hosts are +otherwise compared as written: no percent-decoding, IDNA mapping, or trailing-dot +stripping, so such hosts never match a built-in entry. An entry `scheme://*.domain[:port]` matches any strict +subdomain of `domain` (not `domain` itself). The entry `*` trusts every endpoint. +A destination that does not parse as an absolute `http`/`https` URL, or that contains a +backslash (`\`), is untrusted. +`.prompty` frontmatter MUST NOT be able to grant itself trust. + +Tool connections (§2.9) are out of scope today because no executor attaches ambient +credentials to them; any future handler that does MUST apply the same rule. + ### §2.6 ModelOptions | Property | Type | Description | @@ -815,6 +867,9 @@ flush asynchronously. A failing backend MUST NOT cause the pipeline to fail. | `load` | `(path: string) → PromptAgent` | PromptAgent | | `load_async` | `(path: string) → PromptAgent` | PromptAgent | +Both functions MUST accept host load options `allowedFileRoots` (§2.11) and +`trustedEndpoints` (§2.5.1), named idiomatically per language. + Both functions MUST be traced: emit a `load` span with `inputs = {path: }` and `result = {agent_name: , ...}`. @@ -839,7 +894,11 @@ function load(path): RAISE ValueError("Frontmatter must be a YAML mapping") h. parsed["instructions"] ← body i. RETURN parsed - 5. data ← resolve_references(data, base_dir=parent(path)): + 5. check_endpoint_trust(data, base_dir=parent(path), options): + // §2.5.1 — inspects RAW (unresolved) model.connection + IF credential is ambient AND a file-controlled destination is untrusted: + RAISE untrusted_endpoint + 6. data ← resolve_references(data, base_dir=parent(path)): Walk every value in data recursively: For each string value matching ${protocol:content}: protocol ← lowercase text before first ":" @@ -868,7 +927,7 @@ function load(path): REPLACE with read_file_text(file_path) DEFAULT: Leave value unchanged (unknown protocol) - 6. Apply shorthand expansions: + 7. Apply shorthand expansions: a. IF data["model"] is a string: data["model"] ← {id: data["model"]} b. IF data["template"] is missing: @@ -878,13 +937,13 @@ function load(path): d. For each property in data["inputs"] (if dict form): IF value is a scalar (not a dict): Convert to Property(kind: infer_kind(value), default: value) - 7. Inject kind: + 8. Inject kind: data["kind"] ← "prompt" // .prompty files always produce PromptAgents - 8. agent ← load_typed_agent(data) + 9. agent ← load_typed_agent(data) // Use the generated model loader for the target language // This validates types and produces a PromptAgent instance - 9. RETURN agent + 10. RETURN agent ``` ### §4.3 Dotenv Loading @@ -924,6 +983,11 @@ environment. Environment variables take precedence. | `${env:VAR}` where VAR is not set (no default) | `ValueError` | | `${file:path}` where file does not exist | `FileNotFoundError` | | Invalid YAML in frontmatter | `ValueError` (or YAML parse error) | +| Ambient credential to untrusted endpoint (§2.5.1) | `untrusted_endpoint` | + +The `untrusted_endpoint` message SHOULD name the endpoint and suggest the fixes: +use `${env:VAR}` for the endpoint, add its origin to `trustedEndpoints` / +`PROMPTY_TRUSTED_ENDPOINTS`, or use `kind: reference`. --- diff --git a/web/src/content/docs/core-concepts/file-format.mdx b/web/src/content/docs/core-concepts/file-format.mdx index ad49ec5dd..ff7916a6e 100644 --- a/web/src/content/docs/core-concepts/file-format.mdx +++ b/web/src/content/docs/core-concepts/file-format.mdx @@ -426,6 +426,16 @@ connection: ${file:shared/azure-connection.json} cannot grant itself broader filesystem access. + + ## Shorthand Syntax Reference Prompty supports compact shorthands across several properties. Every shorthand diff --git a/web/src/content/docs/specification/file-format.mdx b/web/src/content/docs/specification/file-format.mdx index 75189c4c7..076793169 100644 --- a/web/src/content/docs/specification/file-format.mdx +++ b/web/src/content/docs/specification/file-format.mdx @@ -132,6 +132,22 @@ Connection types are discriminated by the `kind` field: | `foundry` | `endpoint` | Microsoft Foundry connection | | `oauth` | `endpoint`, `authenticationMode` | OAuth-based authentication | +### §2.5.1 Endpoint Trust + +A `.prompty` file is untrusted input, so it MUST NOT be able to direct **ambient +credentials** (`DefaultAzureCredential` tokens, env API keys) to a destination it chose. +Unless the connection supplies its own literal credential (`apiKey` on `kind: key`), a file-controlled `endpoint` or `tokenUrl` (a literal, a +`${file:}` value, or a used `${env:VAR:default}` fallback) must match a trusted origin, or +loading fails with `untrusted_endpoint`. `kind: reference` and `${env:VAR}` endpoints are +host-controlled and pass. + +Trusted origins: built-ins (`https://api.openai.com`, `https://api.anthropic.com`, +`https://*.openai.azure.com`, `https://*.services.ai.azure.com`, +`https://*.cognitiveservices.azure.com`, `https://login.microsoftonline.com`), the +`trustedEndpoints` load option, and the `PROMPTY_TRUSTED_ENDPOINTS` env var +(comma-separated; `*` trusts all). Matching compares scheme, host, and port only; +`*.domain` matches strict subdomains. + ## §2.6 ModelOptions | Property | Type | Description |