diff --git a/runtime/python/prompty/tests/model/test_vector_conformance.py b/runtime/python/prompty/tests/model/test_vector_conformance.py index e584b97af..1c9ff6f28 100644 --- a/runtime/python/prompty/tests/model/test_vector_conformance.py +++ b/runtime/python/prompty/tests/model/test_vector_conformance.py @@ -5,7 +5,7 @@ import json VECTORS = json.loads( - '[\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "anthropic_enrich_no_dataset_entry_is_noop",\n "stage": "enrichment",\n "provider": "anthropic",\n "input": {\n "id": "claude-sonnet-4-20250514",\n "contextWindow": 200000,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ]\n },\n "expected": {\n "id": "claude-sonnet-4-20250514",\n "contextWindow": 200000,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_dalle3_no_context_window",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "dall-e-3"\n },\n "expected": {\n "id": "dall-e-3",\n "inputModalities": [\n "text"\n ],\n "outputModalities": [\n "image"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_embedding_empty_output_modalities",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "text-embedding-3-small"\n },\n "expected": {\n "id": "text-embedding-3-small",\n "contextWindow": 8191,\n "inputModalities": [\n "text"\n ],\n "outputModalities": []\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_gpt4o",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "gpt-4o"\n },\n "expected": {\n "id": "gpt-4o",\n "contextWindow": 128000,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_gpt4o_mini_longest_prefix",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "gpt-4o-mini-2024-07-18"\n },\n "expected": {\n "id": "gpt-4o-mini-2024-07-18",\n "contextWindow": 128000,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_prefix_requires_token_boundary",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "gpt-45-future"\n },\n "expected": {\n "id": "gpt-45-future"\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_provider_context_window_wins",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "gpt-4o",\n "contextWindow": 999\n },\n "expected": {\n "id": "gpt-4o",\n "contextWindow": 999,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_provider_empty_modalities_win",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "gpt-4o",\n "inputModalities": []\n },\n "expected": {\n "id": "gpt-4o",\n "contextWindow": 128000,\n "inputModalities": [],\n "outputModalities": [\n "text"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_unknown_id_is_noop",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "ft:custom-model:acme::xyz"\n },\n "expected": {\n "id": "ft:custom-model:acme::xyz"\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "anthropic_model_full",\n "stage": "discovery",\n "provider": "anthropic",\n "shape": "model",\n "input": {\n "id": "claude-sonnet-4-20250514",\n "display_name": "Claude Sonnet 4",\n "context_length": 200000,\n "input_modalities": [\n "text",\n "image"\n ],\n "output_modalities": [\n "text"\n ],\n "type": "model"\n },\n "expected": {\n "id": "claude-sonnet-4-20250514",\n "displayName": "Claude Sonnet 4",\n "ownedBy": "anthropic",\n "contextWindow": 200000,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ],\n "additionalProperties": {\n "id": "claude-sonnet-4-20250514",\n "display_name": "Claude Sonnet 4",\n "context_length": 200000,\n "input_modalities": [\n "text",\n "image"\n ],\n "output_modalities": [\n "text"\n ],\n "type": "model"\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "anthropic_model_minimal",\n "stage": "discovery",\n "provider": "anthropic",\n "shape": "model",\n "input": {\n "id": "claude-3-haiku-20240307",\n "type": "model"\n },\n "expected": {\n "id": "claude-3-haiku-20240307",\n "ownedBy": "anthropic",\n "additionalProperties": {\n "id": "claude-3-haiku-20240307",\n "type": "model"\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "foundry_catalog_model",\n "stage": "discovery",\n "provider": "foundry",\n "shape": "catalog",\n "input": {\n "id": "gpt-4",\n "owned_by": "openai",\n "maxContextLength": 8192,\n "status": "succeeded"\n },\n "expected": {\n "id": "gpt-4",\n "ownedBy": "openai",\n "contextWindow": 8192,\n "additionalProperties": {\n "id": "gpt-4",\n "owned_by": "openai",\n "maxContextLength": 8192,\n "status": "succeeded"\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "foundry_deployment_flat_v1",\n "stage": "discovery",\n "provider": "foundry",\n "shape": "deployment",\n "input": {\n "name": "gpt-4o",\n "type": "ModelDeployment",\n "modelName": "gpt-4o",\n "modelVersion": "2024-08-06",\n "modelPublisher": "OpenAI",\n "capabilities": {\n "chatCompletion": "true"\n },\n "sku": {\n "name": "GlobalStandard",\n "capacity": 50\n }\n },\n "expected": {\n "id": "gpt-4o",\n "displayName": "gpt-4o",\n "ownedBy": "OpenAI",\n "additionalProperties": {\n "name": "gpt-4o",\n "type": "ModelDeployment",\n "modelName": "gpt-4o",\n "modelVersion": "2024-08-06",\n "modelPublisher": "OpenAI",\n "capabilities": {\n "chatCompletion": "true"\n },\n "sku": {\n "name": "GlobalStandard",\n "capacity": 50\n }\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "foundry_deployment_nested_arm",\n "stage": "discovery",\n "provider": "foundry",\n "shape": "deployment",\n "input": {\n "name": "my-gpt4",\n "properties": {\n "model": {\n "name": "gpt-4",\n "publisher": "OpenAI",\n "maxContextLength": 8192\n },\n "capabilities": {\n "supportedInputModalities": [\n "text"\n ],\n "supportedOutputModalities": [\n "text"\n ]\n }\n }\n },\n "expected": {\n "id": "my-gpt4",\n "displayName": "gpt-4",\n "ownedBy": "OpenAI",\n "contextWindow": 8192,\n "inputModalities": [\n "text"\n ],\n "outputModalities": [\n "text"\n ],\n "additionalProperties": {\n "name": "my-gpt4",\n "properties": {\n "model": {\n "name": "gpt-4",\n "publisher": "OpenAI",\n "maxContextLength": 8192\n },\n "capabilities": {\n "supportedInputModalities": [\n "text"\n ],\n "supportedOutputModalities": [\n "text"\n ]\n }\n }\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_model_basic",\n "stage": "discovery",\n "provider": "openai",\n "shape": "model",\n "input": {\n "id": "o3-mini",\n "object": "model",\n "created": 1715367049,\n "owned_by": "system"\n },\n "expected": {\n "id": "o3-mini",\n "ownedBy": "system",\n "additionalProperties": {\n "id": "o3-mini",\n "object": "model",\n "created": 1715367049,\n "owned_by": "system"\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_model_finetune_no_owner",\n "stage": "discovery",\n "provider": "openai",\n "shape": "model",\n "input": {\n "id": "ft:custom-model:acme::xyz"\n },\n "expected": {\n "id": "ft:custom-model:acme::xyz",\n "additionalProperties": {\n "id": "ft:custom-model:acme::xyz"\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "basic_load",\n "description": "Load basic.prompty with env var resolution and verify all PromptAgent fields",\n "stage": "load",\n "input": {\n "fixture": "basic.prompty",\n "env": {\n "OPENAI_ENDPOINT": "https://test.openai.com",\n "OPENAI_API_KEY": "sk-test123"\n }\n },\n "expected": {\n "kind": "prompt",\n "name": "basic-prompt",\n "description": "A basic prompt for testing",\n "metadata": {\n "authors": [\n "testauthor"\n ]\n },\n "model": {\n "id": "gpt-4",\n "provider": "openai",\n "apiType": "chat",\n "connection": {\n "kind": "key",\n "endpoint": "https://test.openai.com",\n "apiKey": "sk-test123"\n },\n "options": {\n "temperature": 0.7,\n "maxOutputTokens": 1000\n }\n },\n "inputs": [\n {\n "name": "firstName",\n "kind": "string",\n "default": "Jane"\n },\n {\n "name": "lastName",\n "kind": "string",\n "default": "Doe"\n },\n {\n "name": "question",\n "kind": "string",\n "default": "What is the meaning of life?"\n }\n ],\n "template": {\n "format": {\n "kind": "jinja2"\n },\n "parser": {\n "kind": "prompty"\n }\n },\n "instructions": "system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "connection_types_load",\n "description": "All connection kinds (key, reference, remote, anonymous) load correctly",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "connection-test",\n "model": {\n "id": "gpt-4",\n "connection": {\n "kind": "anonymous",\n "endpoint": "https://localhost:8080"\n }\n }\n }\n },\n "expected": {\n "model": {\n "id": "gpt-4",\n "connection": {\n "kind": "anonymous",\n "endpoint": "https://localhost:8080"\n }\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "embedding_load",\n "description": "Load embedding.prompty and verify apiType=embedding with correct model id",\n "stage": "load",\n "input": {\n "fixture": "embedding.prompty",\n "env": {\n "OPENAI_ENDPOINT": "https://test.openai.com",\n "OPENAI_API_KEY": "sk-test123"\n }\n },\n "expected": {\n "kind": "prompt",\n "name": "embedding",\n "model": {\n "id": "text-embedding-3-small",\n "provider": "openai",\n "apiType": "embedding",\n "connection": {\n "kind": "key",\n "endpoint": "https://test.openai.com",\n "apiKey": "sk-test123"\n }\n },\n "inputs": [\n {\n "name": "text",\n "kind": "string",\n "default": "Hello world"\n }\n ],\n "instructions": "{{text}}"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "empty_frontmatter_body_only",\n "description": "Empty frontmatter with body — instructions extracted, defaults applied",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "empty-fm"\n }\n },\n "expected": {\n "kind": "prompt",\n "name": "empty-fm",\n "model": null,\n "inputs": null,\n "tools": null\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "env_default",\n "description": "${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "env-default-test",\n "model": {\n "id": "gpt-4",\n "connection": {\n "kind": "key",\n "endpoint": "${env:MISSING_VAR:fallback_value}"\n }\n }\n },\n "env": {}\n },\n "expected": {\n "model": {\n "connection": {\n "kind": "key",\n "endpoint": "fallback_value"\n }\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "env_missing_error",\n "description": "${env:NONEXISTENT} with no default and var not set raises an error",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "env-error-test",\n "model": {\n "id": "gpt-4",\n "connection": {\n "kind": "key",\n "endpoint": "${env:NONEXISTENT}"\n }\n }\n },\n "env": {}\n },\n "expected": {\n "error": "Environment variable \'NONEXISTENT\' not set"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "env_resolution",\n "description": "${env:MY_VAR} resolves to the environment variable value when set",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "env-test",\n "model": {\n "id": "gpt-4",\n "connection": {\n "kind": "key",\n "endpoint": "${env:MY_VAR}"\n }\n }\n },\n "env": {\n "MY_VAR": "hello"\n }\n },\n "expected": {\n "model": {\n "connection": {\n "kind": "key",\n "endpoint": "hello"\n }\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "file_resolution",\n "description": "${file:shared_connection.json} resolves to the parsed JSON content of the referenced file",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "file-ref-test",\n "model": {\n "id": "gpt-4",\n "connection": "${file:shared_connection.json}"\n }\n },\n "env": {\n "OPENAI_API_KEY": "shared-key"\n },\n "files": {\n "shared_connection.json": {\n "kind": "key",\n "endpoint": "https://shared.example.com",\n "apiKey": "shared-key"\n }\n }\n },\n "expected": {\n "model": {\n "connection": {\n "kind": "key",\n "endpoint": "https://shared.example.com",\n "apiKey": "shared-key"\n }\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "input_scalar_shorthand",\n "description": "Scalar input values create typed Property with default set",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "scalar-test",\n "model": "gpt-4",\n "inputs": [\n {\n "name": "topic",\n "kind": "string",\n "default": "science"\n },\n {\n "name": "count",\n "kind": "integer",\n "default": 5\n }\n ]\n }\n },\n "expected": {\n "inputs": [\n {\n "name": "topic",\n "kind": "string",\n "default": "science"\n },\n {\n "name": "count",\n "kind": "integer",\n "default": 5\n }\n ]\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "input_validation_default_fill",\n "description": "An input with a default value is filled in when no value is provided",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "default-fill-test",\n "model": "gpt-4",\n "inputs": [\n {\n "name": "name",\n "kind": "string",\n "default": "world"\n }\n ]\n },\n "inputs": {}\n },\n "expected": {\n "validated_inputs": {\n "name": "world"\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "input_validation_example_not_used",\n "description": "An input with example=\'sample\' but no default must NOT use the example at runtime",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "example-not-used-test",\n "model": "gpt-4",\n "inputs": [\n {\n "name": "topic",\n "kind": "string",\n "required": false,\n "example": "sample"\n }\n ]\n },\n "inputs": {}\n },\n "expected": {\n "validated_inputs": {}\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "input_validation_optional_omit",\n "description": "A non-required input with no default and no value is omitted (not an error)",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "optional-omit-test",\n "model": "gpt-4",\n "inputs": [\n {\n "name": "nickname",\n "kind": "string",\n "required": false\n }\n ]\n },\n "inputs": {}\n },\n "expected": {\n "validated_inputs": {}\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "input_validation_required",\n "description": "A required input with no default and no value provided raises ValueError",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "required-input-test",\n "model": "gpt-4",\n "inputs": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n "inputs": {}\n },\n "expected": {\n "error": "Missing required input",\n "error_field": "city"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "instructions_from_body",\n "description": "The markdown body after the closing --- becomes the instructions field",\n "stage": "load",\n "input": {\n "fixture": "basic.prompty",\n "env": {\n "OPENAI_ENDPOINT": "https://test.openai.com",\n "OPENAI_API_KEY": "sk-test123"\n }\n },\n "expected": {\n "instructions": "system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "invalid_frontmatter_error",\n "description": "Malformed YAML in frontmatter raises a parse error",\n "stage": "load",\n "input": {\n "frontmatter_raw": "---\\nname: [invalid\\n---\\nHello"\n },\n "expected": {\n "error": "invalid frontmatter"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "kind_always_prompt",\n "description": "The loader always injects kind=\'prompt\' — .prompty files never specify kind themselves",\n "stage": "load",\n "input": {\n "fixture": "minimal.prompty"\n },\n "expected": {\n "kind": "prompt"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "minimal_load",\n "description": "Load minimal.prompty with model shorthand and bare-minimum frontmatter",\n "stage": "load",\n "input": {\n "fixture": "minimal.prompty"\n },\n "expected": {\n "kind": "prompt",\n "name": "minimal",\n "model": {\n "id": "gpt-4"\n },\n "instructions": "system:\\nHello world.",\n "inputs": null,\n "outputs": null,\n "tools": null\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "missing_file_error",\n "description": "Loading a nonexistent .prompty file raises FileNotFoundError",\n "stage": "load",\n "input": {\n "fixture": "nonexistent.prompty"\n },\n "expected": {\n "error": "FileNotFoundError"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "model_shorthand",\n "description": "String shorthand \'model: gpt-4o\' expands to a Model object with id set",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "test",\n "model": "gpt-4o"\n }\n },\n "expected": {\n "kind": "prompt",\n "name": "test",\n "model": {\n "id": "gpt-4o"\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "template_string_invalid",\n "description": "Template as a bare string is not valid v2 — must be an object with format/parser",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "shorthand-test",\n "model": {\n "id": "gpt-4"\n },\n "template": "jinja2"\n }\n },\n "expected": {\n "error": "Invalid template format"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "tools_custom_load",\n "description": "Load a prompty with an unknown tool kind — falls through to CustomTool",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "custom-tool-test",\n "model": "gpt-4",\n "tools": [\n {\n "name": "my_tool",\n "kind": "my_provider",\n "connection": {\n "kind": "key",\n "endpoint": "https://custom.example.com"\n }\n }\n ]\n }\n },\n "expected": {\n "tools": [\n {\n "name": "my_tool",\n "kind": "my_provider"\n }\n ]\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "tools_function_load",\n "description": "Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings",\n "stage": "load",\n "input": {\n "fixture": "tools_function.prompty"\n },\n "expected": {\n "kind": "prompt",\n "name": "function-tools",\n "model": {\n "id": "gpt-4",\n "apiType": "chat"\n },\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather in a given location",\n "strict": true,\n "parameters": [\n {\n "name": "location",\n "kind": "string",\n "description": "The city and state, e.g. San Francisco, CA"\n },\n {\n "name": "unit",\n "kind": "string",\n "description": "Temperature unit",\n "enumValues": [\n "celsius",\n "fahrenheit"\n ]\n }\n ],\n "bindings": {\n "unit": {\n "input": "preferred_unit"\n }\n }\n }\n ],\n "inputs": [\n {\n "name": "question",\n "kind": "string",\n "default": "What\'s the weather in Paris?"\n },\n {\n "name": "preferred_unit",\n "kind": "string",\n "default": "celsius"\n }\n ],\n "instructions": "system:\\nYou are a helpful assistant with access to tools.\\n\\nuser:\\n{{question}}"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "tools_mcp_load",\n "description": "Load a prompty with an MCP tool — kind, serverName, connection preserved",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "mcp-test",\n "model": "gpt-4",\n "tools": [\n {\n "name": "filesystem",\n "kind": "mcp",\n "serverName": "fs-server",\n "connection": {\n "kind": "reference",\n "name": "my-mcp"\n }\n }\n ]\n }\n },\n "expected": {\n "tools": [\n {\n "name": "filesystem",\n "kind": "mcp",\n "serverName": "fs-server"\n }\n ]\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "tools_openapi_load",\n "description": "Load a prompty with an OpenAPI tool — specification path preserved",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "openapi-test",\n "model": "gpt-4",\n "tools": [\n {\n "name": "weather_api",\n "kind": "openapi",\n "specification": "./weather.json",\n "connection": {\n "kind": "key",\n "endpoint": "https://api.weather.com"\n }\n }\n ]\n }\n },\n "expected": {\n "tools": [\n {\n "name": "weather_api",\n "kind": "openapi",\n "specification": "./weather.json"\n }\n ]\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "tools_prompty_load",\n "description": "Load a prompty with a PromptyTool — path and mode preserved",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "prompty-tool-test",\n "model": "gpt-4",\n "tools": [\n {\n "name": "helper",\n "kind": "prompty",\n "path": "./helper.prompty"\n }\n ]\n }\n },\n "expected": {\n "tools": [\n {\n "name": "helper",\n "kind": "prompty",\n "path": "./helper.prompty"\n }\n ]\n },\n "operation": "load"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "assistant_with_code_block",\n "description": "Code blocks with triple backticks inside a role block are preserved as literal text content",\n "stage": "parse",\n "input": {\n "rendered": "assistant:\\nHere is code:\\n```python\\nprint(\'hello\')\\n```\\n\\nuser:\\nThanks"\n },\n "expected": {\n "messages": [\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "Here is code:\\n```python\\nprint(\'hello\')\\n```"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Thanks"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "consecutive_newlines_between_roles",\n "description": "Extra blank lines between role markers are part of the preceding message\'s content but are stripped by trailing-newline trimming, resulting in clean content",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nHelper\\n\\n\\n\\nuser:\\nHi"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Helper"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "content_trimmed",\n "description": "Leading and trailing newlines are stripped from each message\'s content, but internal whitespace (spaces, indentation) is preserved",\n "stage": "parse",\n "input": {\n "rendered": "system:\\n \\n You are helpful. \\n \\n\\nuser:\\n Hello "\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": " \\n You are helpful. \\n "\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": " Hello "\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "empty_content",\n "description": "A role marker followed immediately by another role marker produces a message with empty string content",\n "stage": "parse",\n "input": {\n "rendered": "system:\\n\\nuser:\\nHello"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": ""\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "markdown_in_content",\n "description": "Markdown formatting (headers, bold, lists) within a role block is preserved verbatim as text content",\n "stage": "parse",\n "input": {\n "rendered": "system:\\n# Title\\n\\nSome **bold** text\\n\\n- item 1\\n- item 2\\n\\nuser:\\nHello"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "# Title\\n\\nSome **bold** text\\n\\n- item 1\\n- item 2"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "multiline_content",\n "description": "Multiple lines within a single role block are preserved as multiline content",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nLine one.\\nLine two.\\nLine three.\\n\\nuser:\\nQuestion"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Line one.\\nLine two.\\nLine three."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Question"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "multiple_assistant_turns",\n "description": "Multiple assistant messages in a conversation",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nYou are helpful.\\n\\nuser:\\nHi\\n\\nassistant:\\nHello!\\n\\nuser:\\nThanks\\n\\nassistant:\\nYou\'re welcome!"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "You are helpful."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "Hello!"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Thanks"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "You\'re welcome!"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "multiple_user_turns",\n "description": "Alternating user and assistant turns produce four messages in conversation order",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nHelper\\n\\nuser:\\nFirst question\\n\\nassistant:\\nFirst answer\\n\\nuser:\\nSecond question"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Helper"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "First question"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "First answer"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Second question"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "no_role_marker_defaults_to_system",\n "description": "Content with no role marker at all defaults to a single system message",\n "stage": "parse",\n "input": {\n "rendered": "Just a plain message with no role marker"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Just a plain message with no role marker"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "role_marker_at_line_start_only",\n "description": "A role name followed by colon mid-line (e.g. \'The user: said\') is NOT treated as a role boundary — only lines matching the boundary regex (role name at line start, colon at end) split messages",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nThe user: said hello\\n\\nuser:\\nHi"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "The user: said hello"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "role_marker_colon_in_content",\n "description": "Colons in content (not at line start) are not treated as role markers",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nHere is a tip: always be kind.\\n\\nuser:\\nTime is 3:30pm"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Here is a tip: always be kind."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Time is 3:30pm"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "single_system",\n "description": "A single system role marker produces one system message",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nYou are helpful."\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "You are helpful."\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "system_user",\n "description": "System and user role markers produce two messages in order",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nYou are helpful.\\n\\nuser:\\nHello"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "You are helpful."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "system_user_assistant",\n "description": "Three role markers (system, user, assistant) produce three messages in order",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nBe concise.\\n\\nuser:\\nHi\\n\\nassistant:\\nHello! How can I help?"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Be concise."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "Hello! How can I help?"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "thread_nonce_expansion",\n "description": "Thread nonce markers emitted by the renderer are expanded into actual conversation messages by the prepare pipeline. The nonce placeholder is replaced by the thread\'s message list, interleaved at the correct position.",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nYou are helpful.\\n\\n__PROMPTY_THREAD_abcd1234_conversation__\\n\\nuser:\\nHi",\n "thread_inputs": {\n "conversation": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "prev question"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "prev answer"\n }\n ]\n }\n ]\n }\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "You are helpful."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "prev question"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "prev answer"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_multiple_text_blocks",\n "description": "Anthropic message response with two text content blocks — concatenates them",\n "stage": "process",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "response": {\n "id": "msg_03ZHFVFLIiCCEzvnrwxXqaNO",\n "type": "message",\n "role": "assistant",\n "model": "claude-sonnet-4-20250514",\n "content": [\n {\n "type": "text",\n "text": "Here is the answer:"\n },\n {\n "type": "text",\n "text": " The weather in Paris is sunny."\n }\n ],\n "stop_reason": "end_turn",\n "stop_sequence": null,\n "usage": {\n "input_tokens": 15,\n "output_tokens": 12\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": "Here is the answer: The weather in Paris is sunny."\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_structured_output",\n "description": "Anthropic text response JSON-parsed when has_outputs is true",\n "stage": "process",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "response": {\n "id": "msg-struct",\n "type": "message",\n "role": "assistant",\n "content": [\n {\n "type": "text",\n "text": "{\\"city\\": \\"Paris\\", \\"temp\\": 22}"\n }\n ],\n "model": "claude-sonnet-4-20250514",\n "stop_reason": "end_turn",\n "stop_sequence": null,\n "usage": {\n "input_tokens": 20,\n "output_tokens": 10\n }\n },\n "has_outputs": true\n },\n "expected": {\n "result": {\n "city": "Paris",\n "temp": 22\n }\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_text",\n "description": "Anthropic message response with a single text content block — extracts the text",\n "stage": "process",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "response": {\n "id": "msg_01XFDUDYJgAACzvnptvVoYEL",\n "type": "message",\n "role": "assistant",\n "model": "claude-sonnet-4-20250514",\n "content": [\n {\n "type": "text",\n "text": "Hello!"\n }\n ],\n "stop_reason": "end_turn",\n "stop_sequence": null,\n "usage": {\n "input_tokens": 10,\n "output_tokens": 3\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": "Hello!"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_tool_use",\n "description": "Anthropic message response with tool_use content block — extracts ToolCall with JSON-stringified input",\n "stage": "process",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "response": {\n "id": "msg_02YGEUEKHgBBDzvnquwWpZMN",\n "type": "message",\n "role": "assistant",\n "model": "claude-sonnet-4-20250514",\n "content": [\n {\n "type": "tool_use",\n "id": "toolu_1",\n "name": "get_weather",\n "input": {\n "city": "Paris"\n }\n }\n ],\n "stop_reason": "tool_use",\n "stop_sequence": null,\n "usage": {\n "input_tokens": 25,\n "output_tokens": 20\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n {\n "id": "toolu_1",\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n }\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_empty_content",\n "description": "Chat completion with empty string content returns empty string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-empty",\n "object": "chat.completion",\n "created": 1700000000,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "",\n "refusal": null\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 5,\n "completion_tokens": 0,\n "total_tokens": 5\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": ""\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_multiple_tool_calls",\n "description": "Chat completion with two parallel tool calls — extracts both ToolCall objects",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-tool2",\n "object": "chat.completion",\n "created": 1700000002,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_a",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n }\n },\n {\n "id": "call_b",\n "type": "function",\n "function": {\n "name": "get_time",\n "arguments": "{\\"timezone\\":\\"Europe/Paris\\"}"\n }\n }\n ],\n "refusal": null\n },\n "finish_reason": "tool_calls"\n }\n ],\n "usage": {\n "prompt_tokens": 30,\n "completion_tokens": 28,\n "total_tokens": 58\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n {\n "id": "call_a",\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n },\n {\n "id": "call_b",\n "name": "get_time",\n "arguments": "{\\"timezone\\":\\"Europe/Paris\\"}"\n }\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_null_content",\n "description": "Chat completion with null content and no tool calls — returns empty string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-null1",\n "object": "chat.completion",\n "created": 1700000006,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "refusal": null\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 10,\n "completion_tokens": 0,\n "total_tokens": 10\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": ""\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_refusal",\n "description": "Chat completion where the model refused the request — returns the refusal string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-refusal1",\n "object": "chat.completion",\n "created": 1700000005,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "refusal": "I cannot help with that"\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 15,\n "completion_tokens": 8,\n "total_tokens": 23\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": "I cannot help with that"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_structured_invalid_json",\n "description": "Chat completion with has_outputs=true but content is not valid JSON — falls back to raw string gracefully",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-struct2",\n "object": "chat.completion",\n "created": 1700000004,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "not valid json",\n "refusal": null\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 20,\n "completion_tokens": 5,\n "total_tokens": 25\n }\n },\n "has_outputs": true\n },\n "expected": {\n "result": "not valid json"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_structured_output",\n "description": "Chat completion with structured output (has_outputs=true) — JSON-parses the content string into an object",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-struct1",\n "object": "chat.completion",\n "created": 1700000003,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "{\\"city\\":\\"Paris\\",\\"temp\\":72}",\n "refusal": null\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 20,\n "completion_tokens": 12,\n "total_tokens": 32\n }\n },\n "has_outputs": true\n },\n "expected": {\n "result": {\n "city": "Paris",\n "temp": 72\n }\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_text_content",\n "description": "Basic chat completion with text content — extracts the message string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-abc123",\n "object": "chat.completion",\n "created": 1700000000,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Hello!",\n "refusal": null\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 10,\n "completion_tokens": 2,\n "total_tokens": 12\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": "Hello!"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_tool_calls",\n "description": "Chat completion with a single tool call — extracts ToolCall list",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-tool1",\n "object": "chat.completion",\n "created": 1700000001,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_1",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n }\n }\n ],\n "refusal": null\n },\n "finish_reason": "tool_calls"\n }\n ],\n "usage": {\n "prompt_tokens": 25,\n "completion_tokens": 15,\n "total_tokens": 40\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n {\n "id": "call_1",\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n }\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "embedding_batch",\n "description": "Batch embedding response with 3 inputs — returns list of 3 vectors",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "embedding",\n "response": {\n "object": "list",\n "data": [\n {\n "object": "embedding",\n "index": 0,\n "embedding": [\n 0.1,\n 0.2\n ]\n },\n {\n "object": "embedding",\n "index": 1,\n "embedding": [\n 0.3,\n 0.4\n ]\n },\n {\n "object": "embedding",\n "index": 2,\n "embedding": [\n 0.5,\n 0.6\n ]\n }\n ],\n "model": "text-embedding-3-small",\n "usage": {\n "prompt_tokens": 15,\n "total_tokens": 15\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n [\n 0.1,\n 0.2\n ],\n [\n 0.3,\n 0.4\n ],\n [\n 0.5,\n 0.6\n ]\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "embedding_single",\n "description": "Single embedding response — extracts the vector directly",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "embedding",\n "response": {\n "object": "list",\n "data": [\n {\n "object": "embedding",\n "index": 0,\n "embedding": [\n 0.1,\n 0.2,\n 0.3\n ]\n }\n ],\n "model": "text-embedding-3-small",\n "usage": {\n "prompt_tokens": 5,\n "total_tokens": 5\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n 0.1,\n 0.2,\n 0.3\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "image_b64",\n "description": "Image generation response with base64 data — extracts the b64_json string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "image",\n "response": {\n "created": 1700000011,\n "data": [\n {\n "b64_json": "base64data",\n "url": null,\n "revised_prompt": null\n }\n ]\n },\n "has_outputs": false\n },\n "expected": {\n "result": "base64data"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "image_revised_prompt",\n "description": "Image generation response with revised_prompt — primary result is the URL, revised_prompt is metadata",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "image",\n "response": {\n "created": 1700000012,\n "data": [\n {\n "url": "https://img.example.com/cat.png",\n "revised_prompt": "A cute cat sitting on a windowsill in warm sunlight"\n }\n ]\n },\n "has_outputs": false\n },\n "expected": {\n "result": "https://img.example.com/cat.png"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "image_url",\n "description": "Image generation response with URL — extracts the image URL",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "image",\n "response": {\n "created": 1700000010,\n "data": [\n {\n "url": "https://img.example.com/generated.png",\n "revised_prompt": null\n }\n ]\n },\n "has_outputs": false\n },\n "expected": {\n "result": "https://img.example.com/generated.png"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_empty_output",\n "description": "Responses API with output_text empty returns empty string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "response": {\n "id": "resp-empty",\n "object": "response",\n "status": "completed",\n "output": [],\n "output_text": "",\n "model": "gpt-4o"\n },\n "has_outputs": false\n },\n "expected": {\n "result": ""\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_structured",\n "description": "Responses API with structured output (has_outputs=true) — JSON-parses output_text into object",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "response": {\n "id": "resp_struct1",\n "object": "response",\n "created_at": 1700000022,\n "model": "gpt-4o-mini",\n "status": "completed",\n "output": [\n {\n "type": "message",\n "id": "msg_002",\n "status": "completed",\n "role": "assistant",\n "content": [\n {\n "type": "output_text",\n "text": "{\\"city\\":\\"Paris\\"}",\n "annotations": []\n }\n ]\n }\n ],\n "output_text": "{\\"city\\":\\"Paris\\"}",\n "usage": {\n "input_tokens": 20,\n "output_tokens": 8,\n "total_tokens": 28\n }\n },\n "has_outputs": true\n },\n "expected": {\n "result": {\n "city": "Paris"\n }\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_text",\n "description": "Responses API with plain text output — extracts output_text",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "response": {\n "id": "resp_abc123",\n "object": "response",\n "created_at": 1700000020,\n "model": "gpt-4o-mini",\n "status": "completed",\n "output": [\n {\n "type": "message",\n "id": "msg_001",\n "status": "completed",\n "role": "assistant",\n "content": [\n {\n "type": "output_text",\n "text": "Hello!",\n "annotations": []\n }\n ]\n }\n ],\n "output_text": "Hello!",\n "usage": {\n "input_tokens": 10,\n "output_tokens": 2,\n "total_tokens": 12\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": "Hello!"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_tool_calls",\n "description": "Responses API with function_call output items — extracts ToolCall list with call_id preserved",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "response": {\n "id": "resp_tool1",\n "object": "response",\n "created_at": 1700000021,\n "model": "gpt-4o-mini",\n "status": "completed",\n "output": [\n {\n "type": "function_call",\n "id": "fc_001",\n "call_id": "call_1",\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}",\n "status": "completed"\n }\n ],\n "output_text": "",\n "usage": {\n "input_tokens": 25,\n "output_tokens": 18,\n "total_tokens": 43\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n {\n "id": "call_1",\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n }\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "conditional_block",\n "description": "Jinja2 if/else block where condition is true",\n "stage": "render",\n "input": {\n "template": "{% if premium %}Welcome VIP!{% else %}Hello!{% endif %}",\n "engine": "jinja2",\n "inputs": {\n "premium": true\n }\n },\n "expected": {\n "rendered": "Welcome VIP!"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "conditional_false",\n "description": "Jinja2 if/else block where condition is false",\n "stage": "render",\n "input": {\n "template": "{% if premium %}Welcome VIP!{% else %}Hello!{% endif %}",\n "engine": "jinja2",\n "inputs": {\n "premium": false\n }\n },\n "expected": {\n "rendered": "Hello!"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "default_filter",\n "description": "Jinja2 default filter provides fallback for missing variables",\n "stage": "render",\n "input": {\n "template": "{{name|default(\\"stranger\\")}}",\n "engine": "jinja2",\n "inputs": {}\n },\n "expected": {\n "rendered": "stranger"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_basic",\n "description": "Jinja2 built-in filter (upper) works in prompty templates",\n "stage": "render",\n "input": {\n "template": "{{name|upper}}",\n "engine": "jinja2",\n "inputs": {\n "name": "hello"\n }\n },\n "expected": {\n "rendered": "HELLO"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_join",\n "description": "Jinja2 join filter joins a list with separator",\n "stage": "render",\n "input": {\n "template": "{{ items | join(\', \') }}",\n "engine": "jinja2",\n "inputs": {\n "items": [\n "a",\n "b",\n "c"\n ]\n }\n },\n "expected": {\n "rendered": "a, b, c"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_length",\n "description": "Jinja2 length filter returns collection size",\n "stage": "render",\n "input": {\n "template": "{{ items | length }}",\n "engine": "jinja2",\n "inputs": {\n "items": [\n 1,\n 2,\n 3,\n 4,\n 5\n ]\n }\n },\n "expected": {\n "rendered": "5"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_lower",\n "description": "Jinja2 lower filter converts to lowercase",\n "stage": "render",\n "input": {\n "template": "{{ name | lower }}",\n "engine": "jinja2",\n "inputs": {\n "name": "HELLO"\n }\n },\n "expected": {\n "rendered": "hello"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_trim",\n "description": "Jinja2 trim filter removes surrounding whitespace",\n "stage": "render",\n "input": {\n "template": "{{ name | trim }}",\n "engine": "jinja2",\n "inputs": {\n "name": " hello "\n }\n },\n "expected": {\n "rendered": "hello"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_upper",\n "description": "Jinja2 upper filter converts to uppercase",\n "stage": "render",\n "input": {\n "template": "{{ name | upper }}",\n "engine": "jinja2",\n "inputs": {\n "name": "hello"\n }\n },\n "expected": {\n "rendered": "HELLO"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "for_loop",\n "description": "Jinja2 for loop iterating over a list of items",\n "stage": "render",\n "input": {\n "template": "Items: {% for item in items %}{{item}} {% endfor %}",\n "engine": "jinja2",\n "inputs": {\n "items": [\n "a",\n "b",\n "c"\n ]\n }\n },\n "expected": {\n "rendered": "Items: a b c "\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "html_not_escaped",\n "description": "HTML content is NOT escaped — prompty templates are not HTML templates",\n "stage": "render",\n "input": {\n "template": "{{content}}",\n "engine": "jinja2",\n "inputs": {\n "content": "bold"\n }\n },\n "expected": {\n "rendered": "bold"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "jinja2_comment",\n "description": "Jinja2 comments are stripped from output",\n "stage": "render",\n "input": {\n "template": "Hello{# this is a comment #} World",\n "engine": "jinja2",\n "inputs": {}\n },\n "expected": {\n "rendered": "Hello World"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "missing_variable_renders_empty",\n "description": "Undefined Jinja2 variable renders as empty string with default Undefined config",\n "stage": "render",\n "input": {\n "template": "Hello {{name}}!",\n "engine": "jinja2",\n "inputs": {}\n },\n "expected": {\n "rendered": "Hello !"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "multiple_variables",\n "description": "Multiple Jinja2 variables of different types including an integer",\n "stage": "render",\n "input": {\n "template": "{{first}} {{last}} is {{age}}",\n "engine": "jinja2",\n "inputs": {\n "first": "Jane",\n "last": "Doe",\n "age": 30\n }\n },\n "expected": {\n "rendered": "Jane Doe is 30"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "mustache_inverted",\n "description": "Mustache inverted section renders when value is falsy",\n "stage": "render",\n "input": {\n "template": "{{^show}}Hidden{{/show}}",\n "engine": "mustache",\n "inputs": {\n "show": false\n }\n },\n "expected": {\n "rendered": "Hidden"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "mustache_loop",\n "description": "Mustache section iterates over list items",\n "stage": "render",\n "input": {\n "template": "{{#items}}{{.}} {{/items}}",\n "engine": "mustache",\n "inputs": {\n "items": [\n "a",\n "b",\n "c"\n ]\n }\n },\n "expected": {\n "rendered": "a b c "\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "mustache_section",\n "description": "Mustache truthy section renders its content",\n "stage": "render",\n "input": {\n "template": "{{#show}}Visible{{/show}}",\n "engine": "mustache",\n "inputs": {\n "show": true\n }\n },\n "expected": {\n "rendered": "Visible"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "mustache_simple",\n "description": "Basic Mustache variable substitution",\n "stage": "render",\n "input": {\n "template": "Hello {{name}}!",\n "engine": "mustache",\n "inputs": {\n "name": "World"\n }\n },\n "expected": {\n "rendered": "Hello World!"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "nested_object",\n "description": "Jinja2 dot notation access into a nested object",\n "stage": "render",\n "input": {\n "template": "{{user.name}} ({{user.email}})",\n "engine": "jinja2",\n "inputs": {\n "user": {\n "name": "Jane",\n "email": "jane@test.com"\n }\n }\n },\n "expected": {\n "rendered": "Jane (jane@test.com)"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "role_markers_preserved",\n "description": "Role markers (system:/user:) are passed through by the renderer — parsing is the parser\'s job",\n "stage": "render",\n "input": {\n "template": "system:\\nYou are helpful.\\n\\nuser:\\n{{question}}",\n "engine": "jinja2",\n "inputs": {\n "question": "Hello"\n }\n },\n "expected": {\n "rendered": "system:\\nYou are helpful.\\n\\nuser:\\nHello"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "simple_substitution",\n "description": "Basic Jinja2 variable substitution with a single variable",\n "stage": "render",\n "input": {\n "template": "Hello {{name}}!",\n "engine": "jinja2",\n "inputs": {\n "name": "World"\n }\n },\n "expected": {\n "rendered": "Hello World!"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "thread_nonce_injection",\n "description": "Thread-kind input is replaced with a nonce marker instead of being rendered directly",\n "stage": "render",\n "input": {\n "template": "system:\\nYou are helpful.\\n\\n{{conversation}}\\n\\nuser:\\n{{question}}",\n "engine": "jinja2",\n "inputs": {\n "question": "Hi",\n "conversation": {\n "_kind": "thread",\n "messages": [\n {\n "role": "user",\n "content": "previous question"\n },\n {\n "role": "assistant",\n "content": "previous answer"\n }\n ]\n }\n }\n },\n "expected": {\n "nonce_pattern": "^system:\\nYou are helpful\\\\.\\n\\n__PROMPTY_THREAD_[a-f0-9]{8}_conversation__\\n\\nuser:\\nHi$"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "whitespace_preserved",\n "description": "Whitespace including indentation and newlines is exactly preserved",\n "stage": "render",\n "input": {\n "template": "line1\\n line2\\n line3",\n "engine": "jinja2",\n "inputs": {}\n },\n "expected": {\n "rendered": "line1\\n line2\\n line3"\n },\n "operation": "render"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "replay",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "max_iterations",\n "stage": "replay",\n "input": {\n "clock": "2026-06-28T00:00:00Z",\n "sessionId": "session-1",\n "turnId": "turn-1",\n "maxIterations": 1\n },\n "expected": [\n "session:session_start:session-1:turn-1",\n "turn:turn_start:0",\n "turn:llm_start:0",\n "turn:llm_complete:0",\n "session:checkpoint_created:session-1:turn-1",\n "turn:permission_requested:0:exec-1-permission",\n "turn:permission_completed:0:true",\n "turn:tool_execution_start:0:add",\n "turn:tool_execution_complete:0:add:true",\n "turn:tool_result:0:add:true",\n "turn:messages_updated:0",\n "turn:error:1:max_iterations",\n "turn:turn_end:1:error",\n "session:session_end:session-1:turn-1:error",\n "summary:session-1:error:turns=1:checkpoints=1"\n ],\n "operation": "replay"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "replay",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "no_tool",\n "stage": "replay",\n "input": {\n "clock": "2026-06-28T00:00:00Z",\n "sessionId": "session-1",\n "turnId": "turn-1",\n "inputs": {\n "name": "Ada"\n },\n "maxIterations": 3\n },\n "expected": [\n "session:session_start:session-1:turn-1",\n "turn:turn_start:0",\n "turn:llm_start:0",\n "turn:llm_complete:0",\n "session:checkpoint_created:session-1:turn-1",\n "turn:turn_end:1:success",\n "session:session_end:session-1:turn-1:success",\n "summary:session-1:success:turns=1:checkpoints=1"\n ],\n "operation": "replay"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "replay",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "permission_denied",\n "stage": "replay",\n "input": {\n "clock": "2026-06-28T00:00:00Z",\n "sessionId": "session-1",\n "turnId": "turn-1"\n },\n "expected": [\n "session:session_start:session-1:turn-1",\n "turn:turn_start:0",\n "turn:llm_start:0",\n "turn:llm_complete:0",\n "session:checkpoint_created:session-1:turn-1",\n "turn:permission_requested:0:exec-1-permission",\n "turn:permission_completed:0:false",\n "turn:tool_result:0:add:false:permission_denied",\n "turn:messages_updated:0",\n "turn:llm_start:1",\n "turn:llm_complete:1",\n "session:checkpoint_created:session-1:turn-1",\n "turn:turn_end:2:success",\n "session:session_end:session-1:turn-1:success",\n "summary:session-1:success:turns=1:checkpoints=2"\n ],\n "operation": "replay"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "replay",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "tool_failure",\n "stage": "replay",\n "input": {\n "clock": "2026-06-28T00:00:00Z",\n "sessionId": "session-1",\n "turnId": "turn-1"\n },\n "expected": [\n "session:session_start:session-1:turn-1",\n "turn:turn_start:0",\n "turn:llm_start:0",\n "turn:llm_complete:0",\n "session:checkpoint_created:session-1:turn-1",\n "turn:permission_requested:0:exec-1-permission",\n "turn:permission_completed:0:true",\n "turn:tool_execution_start:0:fail",\n "turn:tool_execution_complete:0:fail:false:exception",\n "turn:tool_result:0:fail:false:exception",\n "turn:messages_updated:0",\n "turn:llm_start:1",\n "turn:llm_complete:1",\n "session:checkpoint_created:session-1:turn-1",\n "turn:turn_end:2:success",\n "session:session_end:session-1:turn-1:success",\n "summary:session-1:success:turns=1:checkpoints=2"\n ],\n "operation": "replay"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "replay",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "tool_success",\n "stage": "replay",\n "input": {\n "clock": "2026-06-28T00:00:00Z",\n "sessionId": "session-1",\n "turnId": "turn-1"\n },\n "expected": [\n "session:session_start:session-1:turn-1",\n "turn:turn_start:0",\n "turn:llm_start:0",\n "turn:llm_complete:0",\n "session:checkpoint_created:session-1:turn-1",\n "turn:permission_requested:0:exec-1-permission",\n "turn:permission_completed:0:true",\n "turn:tool_execution_start:0:add",\n "turn:tool_execution_complete:0:add:true",\n "turn:tool_result:0:add:true",\n "turn:messages_updated:0",\n "turn:llm_start:1",\n "turn:llm_complete:1",\n "session:checkpoint_created:session-1:turn-1",\n "turn:turn_end:2:success",\n "session:session_end:session-1:turn-1:success",\n "summary:session-1:success:turns=1:checkpoints=2"\n ],\n "operation": "replay"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "assistant_tool_calls_metadata",\n "description": "When the LLM returns tool calls, the assistant message appended to the conversation must have role=\'assistant\', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Compare weather in two cities."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-meta-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_meta_paris",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_meta_london",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_meta_paris",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_meta_london",\n "name": "get_weather",\n "arguments": {\n "city": "London"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_meta_paris",\n "result": "72°F sunny"\n },\n {\n "tool_call_id": "call_meta_london",\n "result": "59°F cloudy"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-meta-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Paris is warmer and sunnier than London today.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "Paris is warmer and sunnier than London today.",\n "iterations": 2,\n "assistant_tool_calls_message": {\n "role": "assistant",\n "content": "",\n "metadata": {\n "tool_calls": [\n {\n "id": "call_meta_paris",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_meta_london",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n }\n }\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "async_tool_function",\n "description": "Agent loop correctly handles async tool functions",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are helpful."\n },\n {\n "role": "user",\n "content": "Look up the data"\n }\n ],\n "tools": [\n {\n "name": "lookup",\n "kind": "function",\n "description": "Look up data",\n "parameters": [\n {\n "name": "query",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "lookup": "returns lookup result"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-async-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_async1",\n "type": "function",\n "function": {\n "name": "lookup",\n "arguments": "{\\"query\\":\\"test\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_async1",\n "name": "lookup",\n "arguments": {\n "query": "test"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_async1",\n "result": "found: test data"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-async-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "I found: test data",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "I found: test data",\n "iterations": 2,\n "total_messages": 6\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "bindings_injected",\n "description": "Tool has a binding {unit: {input: \'preferred_unit\'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "parent_inputs": {\n "preferred_unit": "celsius"\n },\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n },\n {\n "name": "unit",\n "kind": "string",\n "required": false\n }\n ],\n "bindings": {\n "unit": {\n "input": "preferred_unit"\n }\n }\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-binding-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_bind_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_bind_001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "expected_execution_args": {\n "get_weather": {\n "city": "Paris",\n "unit": "celsius"\n }\n },\n "tool_results": [\n {\n "tool_call_id": "call_bind_001",\n "result": "22°C sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-binding-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 22°C and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 22°C and sunny.",\n "iterations": 2,\n "total_messages": 6,\n "notes": "The binding {unit: {input: \'preferred_unit\'}} resolves \'preferred_unit\' from parent_inputs to \'celsius\' and injects it into the tool call arguments before execution, overriding any LLM-provided value."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "cancellation_before_llm",\n "description": "§13.2 Cancellation — Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string"\n },\n "cancel": {\n "cancelled_at": "before_iteration"\n },\n "on_event": true\n },\n "sequence": [],\n "expected": {\n "error": "CancelledError",\n "iterations": 0,\n "total_messages": 2,\n "events": [\n {\n "type": "cancelled",\n "data": {\n "reason": "Cancellation requested before first iteration"\n }\n }\n ],\n "notes": "No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "cancellation_between_iterations",\n "description": "§13.2 Cancellation — Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "cancel": {\n "cancelled_at": "before_iteration_2"\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-cancel-iter-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_cancel_iter001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_cancel_iter001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_cancel_iter001",\n "result": "72°F sunny"\n }\n ]\n }\n ],\n "expected": {\n "error": "CancelledError",\n "iterations": 1,\n "total_messages": 5,\n "events": [\n {\n "type": "status",\n "data": {\n "message": "Starting agent loop"\n }\n },\n {\n "type": "tool_call_start",\n "data": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "type": "tool_result",\n "data": {\n "name": "get_weather",\n "result": "72°F sunny"\n }\n },\n {\n "type": "messages_updated",\n "data": {\n "message_count": 5\n }\n },\n {\n "type": "cancelled",\n "data": {\n "reason": "Cancellation requested before iteration 2"\n }\n }\n ],\n "notes": "Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "cancellation_between_tools",\n "description": "§13.2 Cancellation — LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Compare weather in Paris and London."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "cancel": {\n "cancelled_at": "after_tool_0"\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-cancel-tools-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_cancel_paris",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_cancel_london",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_cancel_paris",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_cancel_london",\n "name": "get_weather",\n "arguments": {\n "city": "London"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_cancel_paris",\n "result": "72°F sunny"\n }\n ]\n }\n ],\n "expected": {\n "error": "CancelledError",\n "iterations": 1,\n "tools_executed": 1,\n "events": [\n {\n "type": "status",\n "data": {\n "message": "Starting agent loop"\n }\n },\n {\n "type": "tool_call_start",\n "data": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "type": "tool_result",\n "data": {\n "name": "get_weather",\n "result": "72°F sunny"\n }\n },\n {\n "type": "cancelled",\n "data": {\n "reason": "Cancellation requested after tool execution"\n }\n }\n ],\n "notes": "Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "context_no_trim_when_fits",\n "description": "§13.3 Context Window — Messages fit within the context budget. No trimming occurs and all messages are preserved.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is 2 + 2?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n },\n "context_budget": 10000\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-ctx-nofit-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "2 + 2 equals 4.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "2 + 2 equals 4.",\n "iterations": 1,\n "total_messages": 3,\n "trimmed_messages": null,\n "notes": "When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "context_preserves_system_messages",\n "description": "§13.3 Context Window — Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "system",\n "content": "Always respond in a formal tone."\n },\n {\n "role": "user",\n "content": "Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail."\n },\n {\n "role": "assistant",\n "content": "Ancient Egypt\'s history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun."\n },\n {\n "role": "user",\n "content": "Now tell me about Greek philosophy covering all major philosophers and their contributions."\n },\n {\n "role": "assistant",\n "content": "Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period."\n },\n {\n "role": "user",\n "content": "What is the weather today?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string"\n },\n "context_budget": 300\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-ctx-sys-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "I would need to know your city to provide the weather.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "I would need to know your city to provide the weather.",\n "iterations": 1,\n "trimmed_messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "system",\n "content": "Always respond in a formal tone."\n },\n {\n "role": "system",\n "content": "[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers."\n },\n {\n "role": "user",\n "content": "What is the weather today?"\n }\n ],\n "notes": "ALL system messages are preserved during context trimming — they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "context_trim_basic",\n "description": "§13.3 Context Window — Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Tell me about the history of Rome. I want a very detailed answer covering all major periods."\n },\n {\n "role": "assistant",\n "content": "Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire."\n },\n {\n "role": "user",\n "content": "Now tell me about the fall of Rome. Again, very detailed with all the key events and dates."\n },\n {\n "role": "assistant",\n "content": "The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453."\n },\n {\n "role": "user",\n "content": "Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events."\n },\n {\n "role": "assistant",\n "content": "After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095."\n },\n {\n "role": "user",\n "content": "Now tell me about the Renaissance period."\n },\n {\n "role": "assistant",\n "content": "The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking."\n },\n {\n "role": "user",\n "content": "Finally, what is the weather in Paris today?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string"\n },\n "context_budget": 500\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-ctx-trim-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_ctx_w001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_ctx_w001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_ctx_w001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-ctx-trim-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 72°F and sunny.",\n "iterations": 2,\n "trimmed_messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "system",\n "content": "[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period."\n },\n {\n "role": "user",\n "content": "Finally, what is the weather in Paris today?"\n }\n ],\n "summary_contains": "User asked",\n "notes": "Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "empty_tool_result",\n "description": "Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content — it must not be skipped or filtered out.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Clear the cache."\n }\n ],\n "tools": [\n {\n "name": "clear_cache",\n "kind": "function",\n "description": "Clear the application cache, returns empty on success",\n "parameters": []\n }\n ],\n "tool_functions": {\n "clear_cache": "returns empty string on success"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-empty-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_empty_001",\n "type": "function",\n "function": {\n "name": "clear_cache",\n "arguments": "{}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_empty_001",\n "name": "clear_cache",\n "arguments": {}\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_empty_001",\n "result": ""\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-empty-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The cache has been cleared successfully.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The cache has been cleared successfully.",\n "iterations": 2,\n "total_messages": 6,\n "tool_result_message": {\n "role": "tool",\n "content": [\n {\n "type": "text",\n "text": ""\n }\n ],\n "metadata": {\n "tool_call_id": "call_empty_001"\n }\n },\n "notes": "Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "events_basic_tool_loop",\n "description": "§13.1 Events — A 2-turn tool-call loop emits the full event lifecycle: status → tool_call_start → tool_result → messages_updated → done.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-evt-basic-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_evt_w001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_evt_w001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_evt_w001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-evt-basic-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 72°F and sunny.",\n "iterations": 2,\n "total_messages": 6,\n "events": [\n {\n "type": "status",\n "data": {\n "message": "Starting agent loop"\n }\n },\n {\n "type": "tool_call_start",\n "data": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "type": "tool_result",\n "data": {\n "name": "get_weather",\n "result": "72°F sunny"\n }\n },\n {\n "type": "messages_updated",\n "data": {\n "message_count": 5\n }\n },\n {\n "type": "done",\n "data": {\n "response": "The weather in Paris is 72°F and sunny."\n }\n }\n ],\n "notes": "Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "events_error_logged",\n "description": "§13.1 Events — A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "raises RuntimeError"\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-evt-err-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_evt_err001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_evt_err001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_evt_err001",\n "result": "Error calling \'get_weather\': RuntimeError: Weather service unavailable"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-evt-err-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Sorry, the weather service is currently unavailable.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "Sorry, the weather service is currently unavailable.",\n "iterations": 2,\n "total_messages": 6,\n "events": [\n {\n "type": "tool_call_start"\n },\n {\n "type": "tool_result"\n },\n {\n "type": "messages_updated"\n },\n {\n "type": "done"\n }\n ],\n "notes": "Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues — this is a success path. Events verify type sequence only (payloads vary by runtime)."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "events_no_tools",\n "description": "§13.1 Events — Single-turn completion with no tool calls. Only the done event is emitted.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is 2 + 2?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-evt-notool-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "2 + 2 equals 4.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "2 + 2 equals 4.",\n "iterations": 1,\n "total_messages": 3,\n "events": [\n {\n "type": "done",\n "data": {\n "response": "2 + 2 equals 4."\n }\n }\n ],\n "notes": "When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "guardrail_all_pass",\n "description": "§13.4 Guardrails — All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "guardrails": {\n "input": {\n "action": "allow"\n },\n "output": {\n "action": "allow"\n },\n "tool": {\n "deny_tools": []\n }\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-guard-pass-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_guard_pass001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_guard_pass001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_guard_pass001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-guard-pass-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 72°F and sunny.",\n "iterations": 2,\n "total_messages": 6,\n "denied_tools": [],\n "notes": "All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "guardrail_input_deny",\n "description": "§13.4 Guardrails — Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "My SSN is 123-45-6789, can you look that up?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n },\n "guardrails": {\n "input": {\n "action": "deny",\n "reason": "Contains PII"\n }\n }\n },\n "sequence": [],\n "expected": {\n "error": "GuardrailError",\n "error_reason": "Contains PII",\n "iterations": 0,\n "total_messages": 2,\n "notes": "Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "guardrail_output_deny",\n "description": "§13.4 Guardrails — Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Tell me something."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n },\n "guardrails": {\n "input": {\n "action": "allow"\n },\n "output": {\n "action": "deny",\n "reason": "Response contains harmful content"\n }\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-guard-out-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Here is some problematic content that the guardrail catches.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "error": "GuardrailError",\n "error_reason": "Response contains harmful content",\n "iterations": 1,\n "notes": "The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "guardrail_tool_deny",\n "description": "§13.4 Guardrails — LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Get the weather and also run the dangerous operation."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "dangerous_tool",\n "kind": "function",\n "description": "A dangerous operation that should be guarded",\n "parameters": [\n {\n "name": "target",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string",\n "dangerous_tool": "returns sensitive data"\n },\n "guardrails": {\n "tool": {\n "deny_tools": [\n "dangerous_tool"\n ],\n "reason": "Tool not authorized"\n }\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-guard-tool-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_guard_weather",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_guard_danger",\n "type": "function",\n "function": {\n "name": "dangerous_tool",\n "arguments": "{\\"target\\": \\"secret_db\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_guard_weather",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_guard_danger",\n "name": "dangerous_tool",\n "arguments": {\n "target": "secret_db"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_guard_weather",\n "result": "72°F sunny"\n },\n {\n "tool_call_id": "call_guard_danger",\n "result": "Tool denied by guardrail: Tool not authorized"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-guard-tool-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny. I was unable to run the dangerous operation as it was not authorized.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 72°F and sunny. I was unable to run the dangerous operation as it was not authorized.",\n "iterations": 2,\n "total_messages": 7,\n "denied_tools": [\n "dangerous_tool"\n ],\n "notes": "The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied — its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "max_iterations_exceeded",\n "description": "LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Keep checking the weather forever."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-loop-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_01",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_01",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_01",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-loop-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_02",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_02",\n "name": "get_weather",\n "arguments": {\n "city": "London"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_02",\n "result": "59°F cloudy"\n }\n ]\n },\n {\n "turn": 3,\n "llm_response": {\n "id": "chatcmpl-loop-003",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_03",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Tokyo\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_03",\n "name": "get_weather",\n "arguments": {\n "city": "Tokyo"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_03",\n "result": "80°F humid"\n }\n ]\n },\n {\n "turn": 4,\n "llm_response": {\n "id": "chatcmpl-loop-004",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_04",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Sydney\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_04",\n "name": "get_weather",\n "arguments": {\n "city": "Sydney"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_04",\n "result": "65°F rainy"\n }\n ]\n },\n {\n "turn": 5,\n "llm_response": {\n "id": "chatcmpl-loop-005",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_05",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Berlin\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_05",\n "name": "get_weather",\n "arguments": {\n "city": "Berlin"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_05",\n "result": "55°F windy"\n }\n ]\n },\n {\n "turn": 6,\n "llm_response": {\n "id": "chatcmpl-loop-006",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_06",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Cairo\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_06",\n "name": "get_weather",\n "arguments": {\n "city": "Cairo"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_06",\n "result": "95°F dry"\n }\n ]\n },\n {\n "turn": 7,\n "llm_response": {\n "id": "chatcmpl-loop-007",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_07",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Mumbai\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_07",\n "name": "get_weather",\n "arguments": {\n "city": "Mumbai"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_07",\n "result": "88°F humid"\n }\n ]\n },\n {\n "turn": 8,\n "llm_response": {\n "id": "chatcmpl-loop-008",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_08",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Moscow\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_08",\n "name": "get_weather",\n "arguments": {\n "city": "Moscow"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_08",\n "result": "30°F snowy"\n }\n ]\n },\n {\n "turn": 9,\n "llm_response": {\n "id": "chatcmpl-loop-009",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_09",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Rio\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_09",\n "name": "get_weather",\n "arguments": {\n "city": "Rio"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_09",\n "result": "85°F tropical"\n }\n ]\n },\n {\n "turn": 10,\n "llm_response": {\n "id": "chatcmpl-loop-010",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_10",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Seoul\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_10",\n "name": "get_weather",\n "arguments": {\n "city": "Seoul"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_10",\n "result": "68°F clear"\n }\n ]\n },\n {\n "turn": 11,\n "llm_response": {\n "id": "chatcmpl-loop-011",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_11",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Toronto\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_11",\n "name": "get_weather",\n "arguments": {\n "city": "Toronto"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_11",\n "result": "40°F foggy"\n }\n ]\n }\n ],\n "expected": {\n "error": "Agent loop exceeded 10 iterations",\n "iterations": 11\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "multi_turn_tool_calls",\n "description": "Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant with access to weather and conversion tools."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris in Celsius?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city (returns Fahrenheit)",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "convert_temperature",\n "kind": "function",\n "description": "Convert a temperature between Fahrenheit and Celsius",\n "parameters": [\n {\n "name": "value",\n "kind": "float",\n "required": true\n },\n {\n "name": "from_unit",\n "kind": "string",\n "required": true\n },\n {\n "name": "to_unit",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string in Fahrenheit",\n "convert_temperature": "returns converted temperature string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-chain-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_weather_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_weather_001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_weather_001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-chain-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_convert_001",\n "type": "function",\n "function": {\n "name": "convert_temperature",\n "arguments": "{\\"value\\": 72.0, \\"from_unit\\": \\"fahrenheit\\", \\"to_unit\\": \\"celsius\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_convert_001",\n "name": "convert_temperature",\n "arguments": {\n "value": 72,\n "from_unit": "fahrenheit",\n "to_unit": "celsius"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_convert_001",\n "result": "22.2°C"\n }\n ]\n },\n {\n "turn": 3,\n "llm_response": {\n "id": "chatcmpl-chain-003",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 22.2°C and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 22.2°C and sunny.",\n "iterations": 3,\n "total_messages": 8\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "multiple_tool_calls_single_turn",\n "description": "Turn 1: LLM returns two parallel tool calls — get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "Compare the weather in Paris and London."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-multi-call-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_paris_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_london_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_paris_001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_london_001",\n "name": "get_weather",\n "arguments": {\n "city": "London"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_paris_001",\n "result": "72°F sunny"\n },\n {\n "tool_call_id": "call_london_001",\n "result": "59°F cloudy"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-multi-call-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Paris is 72°F and sunny, while London is 59°F and cloudy.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "Paris is 72°F and sunny, while London is 59°F and cloudy.",\n "iterations": 2,\n "total_messages": 7,\n "message_sequence": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "Compare the weather in Paris and London."\n },\n {\n "role": "assistant",\n "content": "",\n "metadata": {\n "tool_calls": [\n {\n "id": "call_paris_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_london_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n }\n },\n {\n "role": "tool",\n "content": "72°F sunny",\n "metadata": {\n "tool_call_id": "call_paris_001"\n }\n },\n {\n "role": "tool",\n "content": "59°F cloudy",\n "metadata": {\n "tool_call_id": "call_london_001"\n }\n },\n {\n "role": "assistant",\n "content": "Paris is 72°F and sunny, while London is 59°F and cloudy."\n }\n ]\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "no_tool_calls",\n "description": "LLM returns content immediately with no tool calls — the agent loop completes in a single iteration.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is 2 + 2?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-no-tools-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "2 + 2 equals 4.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "2 + 2 equals 4.",\n "iterations": 1,\n "total_messages": 3\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "parallel_tools_basic",\n "description": "§13.6 Parallel Tools — LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant with access to weather, time, and news tools."\n },\n {\n "role": "user",\n "content": "Give me the weather in Paris, the current time in Tokyo, and the latest news."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "get_time",\n "kind": "function",\n "description": "Get the current time in a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "get_news",\n "kind": "function",\n "description": "Get the latest news headlines",\n "parameters": []\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string",\n "get_time": "returns current time string",\n "get_news": "returns news headlines"\n },\n "parallel_tool_calls": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-parallel-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_par_weather",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_par_time",\n "type": "function",\n "function": {\n "name": "get_time",\n "arguments": "{\\"city\\": \\"Tokyo\\"}"\n }\n },\n {\n "id": "call_par_news",\n "type": "function",\n "function": {\n "name": "get_news",\n "arguments": "{}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_par_weather",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_par_time",\n "name": "get_time",\n "arguments": {\n "city": "Tokyo"\n }\n },\n {\n "id": "call_par_news",\n "name": "get_news",\n "arguments": {}\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_par_weather",\n "result": "72°F sunny"\n },\n {\n "tool_call_id": "call_par_time",\n "result": "3:45 PM JST"\n },\n {\n "tool_call_id": "call_par_news",\n "result": "Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice."\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-parallel-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Here\'s your update: Paris is 72°F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "rust_expected_error": "parallel_tool_calls=true is not supported by the canonical Rust engine",\n "result": "Here\'s your update: Paris is 72°F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.",\n "iterations": 2,\n "total_messages": 8,\n "tool_execution_order": [\n "get_weather",\n "get_time",\n "get_news"\n ],\n "notes": "With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "parallel_tools_with_guardrail_deny",\n "description": "§13.6 Parallel Tools + §13.4 Guardrails — 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Get weather, time, and run the dangerous operation."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "get_time",\n "kind": "function",\n "description": "Get the current time in a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "dangerous_tool",\n "kind": "function",\n "description": "A dangerous operation",\n "parameters": [\n {\n "name": "target",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string",\n "get_time": "returns current time string",\n "dangerous_tool": "returns sensitive data"\n },\n "parallel_tool_calls": true,\n "guardrails": {\n "tool": {\n "deny_tools": [\n "dangerous_tool"\n ],\n "reason": "Not authorized"\n }\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-par-guard-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_pg_weather",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_pg_danger",\n "type": "function",\n "function": {\n "name": "dangerous_tool",\n "arguments": "{\\"target\\": \\"secret_db\\"}"\n }\n },\n {\n "id": "call_pg_time",\n "type": "function",\n "function": {\n "name": "get_time",\n "arguments": "{\\"city\\": \\"Tokyo\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_pg_weather",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_pg_danger",\n "name": "dangerous_tool",\n "arguments": {\n "target": "secret_db"\n }\n },\n {\n "id": "call_pg_time",\n "name": "get_time",\n "arguments": {\n "city": "Tokyo"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_pg_weather",\n "result": "72°F sunny"\n },\n {\n "tool_call_id": "call_pg_danger",\n "result": "Tool denied by guardrail: Not authorized"\n },\n {\n "tool_call_id": "call_pg_time",\n "result": "3:45 PM JST"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-par-guard-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Paris is 72°F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "rust_expected_error": "parallel_tool_calls=true is not supported by the canonical Rust engine",\n "result": "Paris is 72°F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.",\n "iterations": 2,\n "total_messages": 8,\n "denied_tools": [\n "dangerous_tool"\n ],\n "tool_execution_order": [\n "get_weather",\n "get_time"\n ],\n "notes": "The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "single_tool_call",\n "description": "Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-single-tool-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_abc123",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_abc123",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_abc123",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-single-tool-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 72°F and sunny.",\n "iterations": 2,\n "total_messages": 6,\n "message_sequence": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n },\n {\n "role": "assistant",\n "content": "",\n "metadata": {\n "tool_calls": [\n {\n "id": "call_abc123",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n }\n },\n {\n "role": "tool",\n "content": "72°F sunny",\n "metadata": {\n "tool_call_id": "call_abc123"\n }\n },\n {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny."\n }\n ]\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "steering_inject_message",\n "description": "§13.5 Steering — After turn 1 (tool call + result), a steering message is injected before turn 2\'s LLM call. The LLM sees the injected message in context.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "steering": {\n "messages": [\n {\n "inject_before_iteration": 2,\n "role": "user",\n "text": "Actually, focus on the temperature in Celsius."\n }\n ]\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-steer-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_steer_w001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_steer_w001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_steer_w001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-steer-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The temperature in Paris is approximately 22°C and it is sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The temperature in Paris is approximately 22°C and it is sunny.",\n "iterations": 2,\n "total_messages": 7,\n "events": [\n {\n "type": "status",\n "data": {\n "message": "Starting agent loop"\n }\n },\n {\n "type": "tool_call_start",\n "data": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "type": "tool_result",\n "data": {\n "name": "get_weather",\n "result": "72°F sunny"\n }\n },\n {\n "type": "messages_updated",\n "data": {\n "message_count": 5\n }\n },\n {\n "type": "status",\n "data": {\n "message": "Injecting steering message"\n }\n },\n {\n "type": "messages_updated",\n "data": {\n "message_count": 6\n }\n },\n {\n "type": "done",\n "data": {\n "response": "The temperature in Paris is approximately 22°C and it is sunny."\n }\n }\n ],\n "message_sequence": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n },\n {\n "role": "assistant",\n "content": "",\n "metadata": {\n "tool_calls": [\n {\n "id": "call_steer_w001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n }\n },\n {\n "role": "tool",\n "content": "72°F sunny",\n "metadata": {\n "tool_call_id": "call_steer_w001"\n }\n },\n {\n "role": "user",\n "content": "Actually, focus on the temperature in Celsius."\n },\n {\n "role": "assistant",\n "content": "The temperature in Paris is approximately 22°C and it is sunny."\n }\n ],\n "notes": "The steering message is injected as a user message before iteration 2\'s LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "steering_multiple_messages",\n "description": "§13.5 Steering — Two steering messages are queued and both injected before iteration 2. Tests atomic drain — both appear in order.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "steering": {\n "messages": [\n {\n "inject_before_iteration": 2,\n "role": "user",\n "text": "Please respond in French."\n },\n {\n "inject_before_iteration": 2,\n "role": "user",\n "text": "Also include the humidity percentage."\n }\n ]\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-steer-multi-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_steer_m001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_steer_m001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_steer_m001",\n "result": "72°F sunny, 45% humidity"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-steer-multi-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Le temps à Paris est de 22°C et ensoleillé avec une humidité de 45%.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "Le temps à Paris est de 22°C et ensoleillé avec une humidité de 45%.",\n "iterations": 2,\n "total_messages": 8,\n "message_sequence": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n },\n {\n "role": "assistant",\n "content": "",\n "metadata": {\n "tool_calls": [\n {\n "id": "call_steer_m001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n }\n },\n {\n "role": "tool",\n "content": "72°F sunny, 45% humidity",\n "metadata": {\n "tool_call_id": "call_steer_m001"\n }\n },\n {\n "role": "user",\n "content": "Please respond in French."\n },\n {\n "role": "user",\n "content": "Also include the humidity percentage."\n },\n {\n "role": "assistant",\n "content": "Le temps à Paris est de 22°C et ensoleillé avec une humidité de 45%."\n }\n ],\n "notes": "Both steering messages are drained atomically and inserted in order before iteration 2\'s LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles)."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "tool_not_registered_error",\n "description": "LLM calls a tool named \'unknown_tool\' that has no registered handler in tool_functions. The agent loop must raise a ValueError.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Do something with an unknown tool."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-unknown-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_unknown_001",\n "type": "function",\n "function": {\n "name": "unknown_tool",\n "arguments": "{\\"query\\": \\"test\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_unknown_001",\n "name": "unknown_tool",\n "arguments": {\n "query": "test"\n }\n }\n ]\n }\n ],\n "expected": {\n "error": "Tool not registered: unknown_tool",\n "error_type": "ValueError"\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "tool_result_message_format",\n "description": "After executing a tool, the tool result message appended to the conversation must have role=\'tool\', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is the weather?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-format-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_fmt_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_fmt_001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_fmt_001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-format-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "It is 72°F and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "It is 72°F and sunny.",\n "iterations": 2,\n "tool_result_message": {\n "role": "tool",\n "content": [\n {\n "type": "text",\n "text": "72°F sunny"\n }\n ],\n "metadata": {\n "tool_call_id": "call_fmt_001"\n }\n }\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "runTurn",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "cancel_before_context",\n "stage": "turn",\n "input": {\n "messages": [\n {\n "role": "user",\n "content": "Do not invoke the model"\n }\n ],\n "model": [],\n "cancelBeforeRun": true\n },\n "expected": {\n "status": "cancelled",\n "iterations": 0,\n "snapshots": 0,\n "toolResults": 0,\n "eventKinds": [\n "turn_started",\n "turn_cancelled"\n ]\n },\n "operation": "runTurn"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "runTurn",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "delegated_provider_state",\n "stage": "turn",\n "input": {\n "messages": [\n {\n "role": "user",\n "content": "Continue a provider-managed response"\n }\n ],\n "model": [\n {\n "tools": [\n {\n "id": "call-state",\n "name": "echo",\n "arguments": {\n "value": "state"\n }\n }\n ],\n "nextPortability": "delegated",\n "delegatedState": [\n {\n "provider": "openai",\n "kind": "previous_response",\n "id": "resp_123"\n }\n ]\n },\n {\n "output": "continued"\n }\n ],\n "toolOutputs": {\n "call-state": "state"\n }\n },\n "expected": {\n "status": "success",\n "output": "continued",\n "iterations": 2,\n "snapshots": 2,\n "snapshotStablePrefixes": [\n 1,\n 1\n ],\n "snapshotPortability": [\n "portable",\n "delegated"\n ],\n "toolResults": 1,\n "toolResultOrder": [\n "call-state"\n ],\n "commitPortability": "delegated",\n "delegatedState": 1\n },\n "operation": "runTurn"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "runTurn",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "final_output",\n "stage": "turn",\n "input": {\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ],\n "model": [\n {\n "output": "Hello back"\n }\n ]\n },\n "expected": {\n "status": "success",\n "output": "Hello back",\n "iterations": 1,\n "snapshots": 1,\n "snapshotStablePrefixes": [\n 1\n ],\n "toolResults": 0,\n "eventKinds": [\n "turn_started",\n "context_prepared",\n "model_invocation_started",\n "model_invocation_completed",\n "checkpoint_created",\n "turn_committed",\n "post_commit_started",\n "post_commit_completed"\n ]\n },\n "operation": "runTurn"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "runTurn",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "ordered_tool_round",\n "stage": "turn",\n "input": {\n "messages": [\n {\n "role": "user",\n "content": "Add two values"\n }\n ],\n "model": [\n {\n "assistant": "I will use the tools.",\n "tools": [\n {\n "id": "call-a",\n "name": "echo",\n "arguments": {\n "value": "A"\n }\n },\n {\n "id": "call-b",\n "name": "echo",\n "arguments": {\n "value": "B"\n }\n }\n ]\n },\n {\n "output": "A then B"\n }\n ],\n "toolOutputs": {\n "call-a": "A",\n "call-b": "B"\n }\n },\n "expected": {\n "status": "success",\n "output": "A then B",\n "iterations": 2,\n "snapshots": 2,\n "snapshotStablePrefixes": [\n 1,\n 1\n ],\n "toolResults": 2,\n "toolResultOrder": [\n "call-a",\n "call-b"\n ],\n "eventKinds": [\n "turn_started",\n "context_prepared",\n "model_invocation_started",\n "model_invocation_completed",\n "checkpoint_created",\n "permission_requested",\n "permission_resolved",\n "tool_execution_started",\n "tool_execution_completed",\n "checkpoint_created",\n "permission_requested",\n "permission_resolved",\n "tool_execution_started",\n "tool_execution_completed",\n "checkpoint_created",\n "tool_result_committed",\n "tool_result_committed",\n "conversation_updated",\n "checkpoint_created",\n "context_prepared",\n "model_invocation_started",\n "model_invocation_completed",\n "checkpoint_created",\n "turn_committed",\n "post_commit_started",\n "post_commit_completed"\n ]\n },\n "operation": "runTurn"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "runTurn",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "permission_denial_is_model_visible",\n "stage": "turn",\n "input": {\n "messages": [\n {\n "role": "user",\n "content": "Read the protected resource"\n }\n ],\n "model": [\n {\n "tools": [\n {\n "id": "call-denied",\n "name": "protected",\n "arguments": {}\n }\n ]\n },\n {\n "output": "Permission was denied"\n }\n ],\n "denyTools": [\n "protected"\n ]\n },\n "expected": {\n "status": "success",\n "output": "Permission was denied",\n "iterations": 2,\n "snapshots": 2,\n "snapshotStablePrefixes": [\n 1,\n 1\n ],\n "toolResults": 1,\n "toolResultOrder": [\n "call-denied"\n ]\n },\n "operation": "runTurn"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_image_format",\n "description": "§7.5 — Anthropic: ImagePart uses base64 source block with media_type, not image_url format.",\n "stage": "wire",\n "provider": "anthropic",\n "targetApi": "chat",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "model_id": "claude-3",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Describe this image"\n },\n {\n "kind": "image",\n "value": "base64data",\n "mediaType": "image/png"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "maxOutputTokens": 4096\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "claude-3",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Describe this image"\n },\n {\n "type": "image",\n "source": {\n "type": "base64",\n "media_type": "image/png",\n "data": "base64data"\n }\n }\n ]\n }\n ],\n "max_tokens": 4096\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_max_tokens_required",\n "description": "§7.5 — Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).",\n "stage": "wire",\n "provider": "anthropic",\n "targetApi": "chat",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "model_id": "claude-3",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "claude-3",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Hello"\n }\n ]\n }\n ],\n "max_tokens": 4096\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_options",\n "description": "Anthropic: option names map correctly (topK, stopSequences)",\n "stage": "wire",\n "provider": "anthropic",\n "targetApi": "chat",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "model_id": "claude-sonnet-4-20250514",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "temperature": 0.5,\n "topP": 0.9,\n "topK": 40,\n "maxOutputTokens": 2000,\n "stopSequences": [\n "END"\n ]\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "claude-sonnet-4-20250514",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Hi"\n }\n ]\n }\n ],\n "temperature": 0.5,\n "top_p": 0.9,\n "top_k": 40,\n "max_tokens": 2000,\n "stop_sequences": [\n "END"\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_system_separate",\n "description": "§7.5 — Anthropic: system message extracted to top-level \'system\' field; messages array contains only non-system messages.",\n "stage": "wire",\n "provider": "anthropic",\n "targetApi": "chat",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "model_id": "claude-3",\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Be helpful"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "maxOutputTokens": 4096\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "claude-3",\n "system": "Be helpful",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Hello"\n }\n ]\n }\n ],\n "max_tokens": 4096\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_tool_wire",\n "description": "Anthropic: tools use input_schema (not parameters), name at top level",\n "stage": "wire",\n "provider": "anthropic",\n "targetApi": "chat",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "model_id": "claude-sonnet-4-20250514",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Weather?"\n }\n ]\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get weather",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "claude-sonnet-4-20250514",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Weather?"\n }\n ]\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "description": "Get weather",\n "input_schema": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string"\n }\n },\n "required": [\n "city"\n ]\n }\n }\n ],\n "max_tokens": 4096\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_audio_mp3",\n "description": "§7.1.2 — AudioPart with mediaType audio/mpeg MUST map to format \'mp3\', not \'mpeg\'.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "audio",\n "value": "base64data",\n "mediaType": "audio/mpeg"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "input_audio",\n "input_audio": {\n "data": "base64data",\n "format": "mp3"\n }\n }\n ]\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_audio_part",\n "description": "§7.1.2 — AudioPart with mediaType audio/wav maps to input_audio with format \'wav\'.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "audio",\n "value": "base64data",\n "mediaType": "audio/wav"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "input_audio",\n "input_audio": {\n "data": "base64data",\n "format": "wav"\n }\n }\n ]\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_image_base64",\n "description": "Base64 data URI image part preserved as-is in image_url",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Describe this"\n },\n {\n "kind": "image",\n "value": "data:image/png;base64,iVBOR..."\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Describe this"\n },\n {\n "type": "image_url",\n "image_url": {\n "url": "data:image/png;base64,iVBOR..."\n }\n }\n ]\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_image_part",\n "description": "Image content part converted to OpenAI image_url format",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "What\'s in this image?"\n },\n {\n "kind": "image",\n "value": "https://example.com/photo.jpg"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "What\'s in this image?"\n },\n {\n "type": "image_url",\n "image_url": {\n "url": "https://example.com/photo.jpg"\n }\n }\n ]\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_multipart_content",\n "description": "§7.1.1/§7.1.2 — Message with TextPart + ImagePart MUST produce an array of typed content blocks.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Describe this image"\n },\n {\n "kind": "image",\n "value": "https://img.png"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Describe this image"\n },\n {\n "type": "image_url",\n "image_url": {\n "url": "https://img.png"\n }\n }\n ]\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_simple",\n "description": "§7.1.7 — Two messages (system + user), no tools, no options. Minimal chat request.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Be helpful"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "system",\n "content": "Be helpful"\n },\n {\n "role": "user",\n "content": "Hello"\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_single_text_optimized",\n "description": "§7.1.1 — Message with exactly 1 TextPart MUST produce a plain string content, not an array.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "What is 2+2?"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "What is 2+2?"\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_with_options",\n "description": "§7.1.5 — All standard ModelOptions mapped to OpenAI parameter names.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "temperature": 0.5,\n "maxOutputTokens": 100,\n "topP": 0.9,\n "frequencyPenalty": 0.1,\n "presencePenalty": 0.2,\n "seed": 42,\n "stopSequences": [\n "END"\n ]\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ],\n "temperature": 0.5,\n "max_completion_tokens": 100,\n "top_p": 0.9,\n "frequency_penalty": 0.1,\n "presence_penalty": 0.2,\n "seed": 42,\n "stop": [\n "END"\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "embedding_wire",\n "description": "§7.2 — Embedding request: text extracted from messages, single string input (not array).",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "embedding",\n "input": {\n "provider": "openai",\n "apiType": "embedding",\n "model_id": "text-embedding-3-small",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello world"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "text-embedding-3-small",\n "input": "Hello world"\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "image_wire",\n "description": "§7.3 — Image generation request: prompt extracted from last user message.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "image",\n "input": {\n "provider": "openai",\n "apiType": "image",\n "model_id": "dall-e-3",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "A cat"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "dall-e-3",\n "prompt": "A cat"\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "kind_to_json_type_mapping",\n "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Test"\n }\n ]\n }\n ],\n "tools": [\n {\n "kind": "function",\n "name": "test_types",\n "description": "Tests all type mappings",\n "parameters": [\n {\n "name": "a_string",\n "kind": "string"\n },\n {\n "name": "an_integer",\n "kind": "integer"\n },\n {\n "name": "a_float",\n "kind": "float"\n },\n {\n "name": "a_boolean",\n "kind": "boolean"\n },\n {\n "name": "an_array",\n "kind": "array",\n "items": {\n "kind": "string"\n }\n },\n {\n "name": "an_object",\n "kind": "object"\n }\n ]\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Test"\n }\n ],\n "tools": [\n {\n "type": "function",\n "function": {\n "name": "test_types",\n "description": "Tests all type mappings",\n "parameters": {\n "type": "object",\n "properties": {\n "a_string": {\n "type": "string"\n },\n "an_integer": {\n "type": "integer"\n },\n "a_float": {\n "type": "number"\n },\n "a_boolean": {\n "type": "boolean"\n },\n "an_array": {\n "type": "array",\n "items": {\n "type": "string"\n }\n },\n "an_object": {\n "type": "object"\n }\n }\n }\n }\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "options_additional_properties",\n "description": "§7.1.5 — additionalProperties from ModelOptions MUST be merged into the request as top-level keys.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "additionalProperties": {\n "logprobs": true\n }\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ],\n "logprobs": true\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "options_max_completion_tokens",\n "description": "§7.1.5 — maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "maxOutputTokens": 500\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ],\n "max_completion_tokens": 500\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "options_stop_sequences",\n "description": "§7.1.5 — stopSequences MUST map to \'stop\' in the OpenAI wire format.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "stopSequences": [\n ".",\n "!"\n ]\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ],\n "stop": [\n ".",\n "!"\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_simple",\n "description": "Responses API: system becomes instructions, messages become input items",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "responses",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "model_id": "gpt-4o",\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Be helpful"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o",\n "instructions": "Be helpful",\n "input": [\n {\n "role": "user",\n "content": "Hello"\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_structured_output",\n "description": "Responses API: structured output uses text.format.json_schema",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "responses",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "model_id": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Give me data"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": [\n {\n "name": "city",\n "kind": "string"\n },\n {\n "name": "temp",\n "kind": "integer"\n }\n ]\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o",\n "input": [\n {\n "role": "user",\n "content": "Give me data"\n }\n ],\n "text": {\n "format": {\n "type": "json_schema",\n "name": "structured_output",\n "schema": {\n "type": "object",\n "properties": {\n "city": {\n "type": [\n "string",\n "null"\n ]\n },\n "temp": {\n "type": [\n "integer",\n "null"\n ]\n }\n },\n "required": [\n "city",\n "temp"\n ],\n "additionalProperties": false\n },\n "strict": true\n }\n }\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_with_tools",\n "description": "Responses API: tools use flat format (no nested function key)",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "responses",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "model_id": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Weather?"\n }\n ]\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get weather",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o",\n "input": [\n {\n "role": "user",\n "content": "Weather?"\n }\n ],\n "tools": [\n {\n "type": "function",\n "name": "get_weather",\n "description": "Get weather",\n "parameters": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string"\n }\n },\n "required": [\n "city"\n ]\n }\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "structured_output",\n "description": "§7.1.6 — outputs converted to response_format with json_schema, strict=true, additionalProperties=false.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "What is the weather in Paris?"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n },\n {\n "name": "temp",\n "kind": "integer",\n "required": true\n }\n ]\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "response_format": {\n "type": "json_schema",\n "json_schema": {\n "name": "structured_output",\n "strict": true,\n "schema": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string"\n },\n "temp": {\n "type": "integer"\n }\n },\n "required": [\n "city",\n "temp"\n ],\n "additionalProperties": false\n }\n }\n }\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "tools_bindings_stripped",\n "description": "§7.1.3 — Parameters listed in bindings MUST be stripped from wire tools (properties AND required).",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Get the weather"\n }\n ]\n }\n ],\n "tools": [\n {\n "kind": "function",\n "name": "get_weather",\n "description": "Get weather",\n "parameters": [\n {\n "name": "location",\n "kind": "string",\n "required": true\n },\n {\n "name": "unit",\n "kind": "string",\n "required": true\n }\n ],\n "bindings": {\n "unit": {\n "input": "preferred_unit"\n }\n }\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Get the weather"\n }\n ],\n "tools": [\n {\n "type": "function",\n "function": {\n "name": "get_weather",\n "description": "Get weather",\n "parameters": {\n "type": "object",\n "properties": {\n "location": {\n "type": "string"\n }\n },\n "required": [\n "location"\n ]\n }\n }\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "tools_function_wire",\n "description": "§7.1.3 — FunctionTool projected as OpenAI function definition with JSON Schema parameters.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "What is the weather?"\n }\n ]\n }\n ],\n "tools": [\n {\n "kind": "function",\n "name": "get_weather",\n "description": "Get weather",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "What is the weather?"\n }\n ],\n "tools": [\n {\n "type": "function",\n "function": {\n "name": "get_weather",\n "description": "Get weather",\n "parameters": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string"\n }\n },\n "required": [\n "city"\n ]\n }\n }\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "tools_null_when_empty",\n "description": "§7.1.3 — When no tools are defined, the tools key MUST be absent from the request entirely.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "tools_strict_mode",\n "description": "§7.1.3 — FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Get the weather"\n }\n ]\n }\n ],\n "tools": [\n {\n "kind": "function",\n "name": "get_weather",\n "description": "Get weather",\n "strict": true,\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Get the weather"\n }\n ],\n "tools": [\n {\n "type": "function",\n "function": {\n "name": "get_weather",\n "description": "Get weather",\n "strict": true,\n "parameters": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string"\n }\n },\n "required": [\n "city"\n ],\n "additionalProperties": false\n }\n }\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n }\n]' + '[\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "anthropic_enrich_no_dataset_entry_is_noop",\n "stage": "enrichment",\n "provider": "anthropic",\n "input": {\n "id": "claude-sonnet-4-20250514",\n "contextWindow": 200000,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ]\n },\n "expected": {\n "id": "claude-sonnet-4-20250514",\n "contextWindow": 200000,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_dalle3_no_context_window",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "dall-e-3"\n },\n "expected": {\n "id": "dall-e-3",\n "inputModalities": [\n "text"\n ],\n "outputModalities": [\n "image"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_embedding_empty_output_modalities",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "text-embedding-3-small"\n },\n "expected": {\n "id": "text-embedding-3-small",\n "contextWindow": 8191,\n "inputModalities": [\n "text"\n ],\n "outputModalities": []\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_gpt4o",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "gpt-4o"\n },\n "expected": {\n "id": "gpt-4o",\n "contextWindow": 128000,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_gpt4o_mini_longest_prefix",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "gpt-4o-mini-2024-07-18"\n },\n "expected": {\n "id": "gpt-4o-mini-2024-07-18",\n "contextWindow": 128000,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_prefix_requires_token_boundary",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "gpt-45-future"\n },\n "expected": {\n "id": "gpt-45-future"\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_provider_context_window_wins",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "gpt-4o",\n "contextWindow": 999\n },\n "expected": {\n "id": "gpt-4o",\n "contextWindow": 999,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_provider_empty_modalities_win",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "gpt-4o",\n "inputModalities": []\n },\n "expected": {\n "id": "gpt-4o",\n "contextWindow": 128000,\n "inputModalities": [],\n "outputModalities": [\n "text"\n ]\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "enrich",\n "params": {\n "base": "ModelInfo"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_enrich_unknown_id_is_noop",\n "stage": "enrichment",\n "provider": "openai",\n "input": {\n "id": "ft:custom-model:acme::xyz"\n },\n "expected": {\n "id": "ft:custom-model:acme::xyz"\n },\n "operation": "enrich"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "anthropic_model_full",\n "stage": "discovery",\n "provider": "anthropic",\n "shape": "model",\n "input": {\n "id": "claude-sonnet-4-20250514",\n "display_name": "Claude Sonnet 4",\n "context_length": 200000,\n "input_modalities": [\n "text",\n "image"\n ],\n "output_modalities": [\n "text"\n ],\n "type": "model"\n },\n "expected": {\n "id": "claude-sonnet-4-20250514",\n "displayName": "Claude Sonnet 4",\n "ownedBy": "anthropic",\n "contextWindow": 200000,\n "inputModalities": [\n "text",\n "image"\n ],\n "outputModalities": [\n "text"\n ],\n "additionalProperties": {\n "id": "claude-sonnet-4-20250514",\n "display_name": "Claude Sonnet 4",\n "context_length": 200000,\n "input_modalities": [\n "text",\n "image"\n ],\n "output_modalities": [\n "text"\n ],\n "type": "model"\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "anthropic_model_minimal",\n "stage": "discovery",\n "provider": "anthropic",\n "shape": "model",\n "input": {\n "id": "claude-3-haiku-20240307",\n "type": "model"\n },\n "expected": {\n "id": "claude-3-haiku-20240307",\n "ownedBy": "anthropic",\n "additionalProperties": {\n "id": "claude-3-haiku-20240307",\n "type": "model"\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "foundry_catalog_model",\n "stage": "discovery",\n "provider": "foundry",\n "shape": "catalog",\n "input": {\n "id": "gpt-4",\n "owned_by": "openai",\n "maxContextLength": 8192,\n "status": "succeeded"\n },\n "expected": {\n "id": "gpt-4",\n "ownedBy": "openai",\n "contextWindow": 8192,\n "additionalProperties": {\n "id": "gpt-4",\n "owned_by": "openai",\n "maxContextLength": 8192,\n "status": "succeeded"\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "foundry_deployment_flat_v1",\n "stage": "discovery",\n "provider": "foundry",\n "shape": "deployment",\n "input": {\n "name": "gpt-4o",\n "type": "ModelDeployment",\n "modelName": "gpt-4o",\n "modelVersion": "2024-08-06",\n "modelPublisher": "OpenAI",\n "capabilities": {\n "chatCompletion": "true"\n },\n "sku": {\n "name": "GlobalStandard",\n "capacity": 50\n }\n },\n "expected": {\n "id": "gpt-4o",\n "displayName": "gpt-4o",\n "ownedBy": "OpenAI",\n "additionalProperties": {\n "name": "gpt-4o",\n "type": "ModelDeployment",\n "modelName": "gpt-4o",\n "modelVersion": "2024-08-06",\n "modelPublisher": "OpenAI",\n "capabilities": {\n "chatCompletion": "true"\n },\n "sku": {\n "name": "GlobalStandard",\n "capacity": 50\n }\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "foundry_deployment_nested_arm",\n "stage": "discovery",\n "provider": "foundry",\n "shape": "deployment",\n "input": {\n "name": "my-gpt4",\n "properties": {\n "model": {\n "name": "gpt-4",\n "publisher": "OpenAI",\n "maxContextLength": 8192\n },\n "capabilities": {\n "supportedInputModalities": [\n "text"\n ],\n "supportedOutputModalities": [\n "text"\n ]\n }\n }\n },\n "expected": {\n "id": "my-gpt4",\n "displayName": "gpt-4",\n "ownedBy": "OpenAI",\n "contextWindow": 8192,\n "inputModalities": [\n "text"\n ],\n "outputModalities": [\n "text"\n ],\n "additionalProperties": {\n "name": "my-gpt4",\n "properties": {\n "model": {\n "name": "gpt-4",\n "publisher": "OpenAI",\n "maxContextLength": 8192\n },\n "capabilities": {\n "supportedInputModalities": [\n "text"\n ],\n "supportedOutputModalities": [\n "text"\n ]\n }\n }\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_model_basic",\n "stage": "discovery",\n "provider": "openai",\n "shape": "model",\n "input": {\n "id": "o3-mini",\n "object": "model",\n "created": 1715367049,\n "owned_by": "system"\n },\n "expected": {\n "id": "o3-mini",\n "ownedBy": "system",\n "additionalProperties": {\n "id": "o3-mini",\n "object": "model",\n "created": 1715367049,\n "owned_by": "system"\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "DiscoveryConformance",\n "operation": "mapModel",\n "params": {\n "raw": "unknown"\n },\n "returns": "ModelInfo",\n "vector": {\n "name": "openai_model_finetune_no_owner",\n "stage": "discovery",\n "provider": "openai",\n "shape": "model",\n "input": {\n "id": "ft:custom-model:acme::xyz"\n },\n "expected": {\n "id": "ft:custom-model:acme::xyz",\n "additionalProperties": {\n "id": "ft:custom-model:acme::xyz"\n }\n },\n "operation": "mapModel"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "basic_load",\n "description": "Load basic.prompty with env var resolution and verify all PromptAgent fields",\n "stage": "load",\n "input": {\n "fixture": "basic.prompty",\n "env": {\n "OPENAI_ENDPOINT": "https://test.openai.com",\n "OPENAI_API_KEY": "sk-test123"\n }\n },\n "expected": {\n "kind": "prompt",\n "name": "basic-prompt",\n "description": "A basic prompt for testing",\n "metadata": {\n "authors": [\n "testauthor"\n ]\n },\n "model": {\n "id": "gpt-4",\n "provider": "openai",\n "apiType": "chat",\n "connection": {\n "kind": "key",\n "endpoint": "https://test.openai.com",\n "apiKey": "sk-test123"\n },\n "options": {\n "temperature": 0.7,\n "maxOutputTokens": 1000\n }\n },\n "inputs": [\n {\n "name": "firstName",\n "kind": "string",\n "default": "Jane"\n },\n {\n "name": "lastName",\n "kind": "string",\n "default": "Doe"\n },\n {\n "name": "question",\n "kind": "string",\n "default": "What is the meaning of life?"\n }\n ],\n "template": {\n "format": {\n "kind": "jinja2"\n },\n "parser": {\n "kind": "prompty"\n }\n },\n "instructions": "system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "connection_types_load",\n "description": "All connection kinds (key, reference, remote, anonymous) load correctly",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "connection-test",\n "model": {\n "id": "gpt-4",\n "connection": {\n "kind": "anonymous",\n "endpoint": "https://localhost:8080"\n }\n }\n }\n },\n "expected": {\n "model": {\n "id": "gpt-4",\n "connection": {\n "kind": "anonymous",\n "endpoint": "https://localhost:8080"\n }\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "embedding_load",\n "description": "Load embedding.prompty and verify apiType=embedding with correct model id",\n "stage": "load",\n "input": {\n "fixture": "embedding.prompty",\n "env": {\n "OPENAI_ENDPOINT": "https://test.openai.com",\n "OPENAI_API_KEY": "sk-test123"\n }\n },\n "expected": {\n "kind": "prompt",\n "name": "embedding",\n "model": {\n "id": "text-embedding-3-small",\n "provider": "openai",\n "apiType": "embedding",\n "connection": {\n "kind": "key",\n "endpoint": "https://test.openai.com",\n "apiKey": "sk-test123"\n }\n },\n "inputs": [\n {\n "name": "text",\n "kind": "string",\n "default": "Hello world"\n }\n ],\n "instructions": "{{text}}"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "empty_frontmatter_body_only",\n "description": "Empty frontmatter with body — instructions extracted, defaults applied",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "empty-fm"\n }\n },\n "expected": {\n "kind": "prompt",\n "name": "empty-fm",\n "model": null,\n "inputs": null,\n "tools": null\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "env_default",\n "description": "${env:MISSING_VAR:fallback_value} resolves to the default when the var is not set",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "env-default-test",\n "model": {\n "id": "gpt-4",\n "connection": {\n "kind": "key",\n "endpoint": "${env:MISSING_VAR:fallback_value}"\n }\n }\n },\n "env": {}\n },\n "expected": {\n "model": {\n "connection": {\n "kind": "key",\n "endpoint": "fallback_value"\n }\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "env_missing_error",\n "description": "${env:NONEXISTENT} with no default and var not set raises an error",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "env-error-test",\n "model": {\n "id": "gpt-4",\n "connection": {\n "kind": "key",\n "endpoint": "${env:NONEXISTENT}"\n }\n }\n },\n "env": {}\n },\n "expected": {\n "error": "Environment variable \'NONEXISTENT\' not set"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "env_resolution",\n "description": "${env:MY_VAR} resolves to the environment variable value when set",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "env-test",\n "model": {\n "id": "gpt-4",\n "connection": {\n "kind": "key",\n "endpoint": "${env:MY_VAR}"\n }\n }\n },\n "env": {\n "MY_VAR": "hello"\n }\n },\n "expected": {\n "model": {\n "connection": {\n "kind": "key",\n "endpoint": "hello"\n }\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "file_resolution",\n "description": "${file:shared_connection.json} resolves to the parsed JSON content of the referenced file",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "file-ref-test",\n "model": {\n "id": "gpt-4",\n "connection": "${file:shared_connection.json}"\n }\n },\n "env": {\n "OPENAI_API_KEY": "shared-key"\n },\n "files": {\n "shared_connection.json": {\n "kind": "key",\n "endpoint": "https://shared.example.com",\n "apiKey": "shared-key"\n }\n }\n },\n "expected": {\n "model": {\n "connection": {\n "kind": "key",\n "endpoint": "https://shared.example.com",\n "apiKey": "shared-key"\n }\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "input_scalar_shorthand",\n "description": "Scalar input values create typed Property with default set",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "scalar-test",\n "model": "gpt-4",\n "inputs": [\n {\n "name": "topic",\n "kind": "string",\n "default": "science"\n },\n {\n "name": "count",\n "kind": "integer",\n "default": 5\n }\n ]\n }\n },\n "expected": {\n "inputs": [\n {\n "name": "topic",\n "kind": "string",\n "default": "science"\n },\n {\n "name": "count",\n "kind": "integer",\n "default": 5\n }\n ]\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "input_validation_default_fill",\n "description": "An input with a default value is filled in when no value is provided",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "default-fill-test",\n "model": "gpt-4",\n "inputs": [\n {\n "name": "name",\n "kind": "string",\n "default": "world"\n }\n ]\n },\n "inputs": {}\n },\n "expected": {\n "validated_inputs": {\n "name": "world"\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "input_validation_example_not_used",\n "description": "An input with example=\'sample\' but no default must NOT use the example at runtime",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "example-not-used-test",\n "model": "gpt-4",\n "inputs": [\n {\n "name": "topic",\n "kind": "string",\n "required": false,\n "example": "sample"\n }\n ]\n },\n "inputs": {}\n },\n "expected": {\n "validated_inputs": {}\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "input_validation_optional_omit",\n "description": "A non-required input with no default and no value is omitted (not an error)",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "optional-omit-test",\n "model": "gpt-4",\n "inputs": [\n {\n "name": "nickname",\n "kind": "string",\n "required": false\n }\n ]\n },\n "inputs": {}\n },\n "expected": {\n "validated_inputs": {}\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "input_validation_required",\n "description": "A required input with no default and no value provided raises ValueError",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "required-input-test",\n "model": "gpt-4",\n "inputs": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n "inputs": {}\n },\n "expected": {\n "error": "Missing required input",\n "error_field": "city"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "instructions_from_body",\n "description": "The markdown body after the closing --- becomes the instructions field",\n "stage": "load",\n "input": {\n "fixture": "basic.prompty",\n "env": {\n "OPENAI_ENDPOINT": "https://test.openai.com",\n "OPENAI_API_KEY": "sk-test123"\n }\n },\n "expected": {\n "instructions": "system:\\nYou are an AI assistant who helps people find information.\\n\\n# Customer\\nYou are helping {{firstName}} {{lastName}} to find answers to their questions.\\n\\nuser:\\n{{question}}"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "invalid_frontmatter_error",\n "description": "Malformed YAML in frontmatter raises a parse error",\n "stage": "load",\n "input": {\n "frontmatter_raw": "---\\nname: [invalid\\n---\\nHello"\n },\n "expected": {\n "error": "invalid frontmatter"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "kind_always_prompt",\n "description": "The loader always injects kind=\'prompt\' — .prompty files never specify kind themselves",\n "stage": "load",\n "input": {\n "fixture": "minimal.prompty"\n },\n "expected": {\n "kind": "prompt"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "minimal_load",\n "description": "Load minimal.prompty with model shorthand and bare-minimum frontmatter",\n "stage": "load",\n "input": {\n "fixture": "minimal.prompty"\n },\n "expected": {\n "kind": "prompt",\n "name": "minimal",\n "model": {\n "id": "gpt-4"\n },\n "instructions": "system:\\nHello world.",\n "inputs": null,\n "outputs": null,\n "tools": null\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "missing_file_error",\n "description": "Loading a nonexistent .prompty file raises FileNotFoundError",\n "stage": "load",\n "input": {\n "fixture": "nonexistent.prompty"\n },\n "expected": {\n "error": "FileNotFoundError"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "model_shorthand",\n "description": "String shorthand \'model: gpt-4o\' expands to a Model object with id set",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "test",\n "model": "gpt-4o"\n }\n },\n "expected": {\n "kind": "prompt",\n "name": "test",\n "model": {\n "id": "gpt-4o"\n }\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "template_string_invalid",\n "description": "Template as a bare string is not valid v2 — must be an object with format/parser",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "shorthand-test",\n "model": {\n "id": "gpt-4"\n },\n "template": "jinja2"\n }\n },\n "expected": {\n "error": "Invalid template format"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "tools_custom_load",\n "description": "Load a prompty with an unknown tool kind — falls through to CustomTool",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "custom-tool-test",\n "model": "gpt-4",\n "tools": [\n {\n "name": "my_tool",\n "kind": "my_provider",\n "connection": {\n "kind": "key",\n "endpoint": "https://custom.example.com"\n }\n }\n ]\n }\n },\n "expected": {\n "tools": [\n {\n "name": "my_tool",\n "kind": "my_provider"\n }\n ]\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "tools_function_load",\n "description": "Load tools_function.prompty and verify FunctionTool with parameters, strict, and bindings",\n "stage": "load",\n "input": {\n "fixture": "tools_function.prompty"\n },\n "expected": {\n "kind": "prompt",\n "name": "function-tools",\n "model": {\n "id": "gpt-4",\n "apiType": "chat"\n },\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather in a given location",\n "strict": true,\n "parameters": [\n {\n "name": "location",\n "kind": "string",\n "description": "The city and state, e.g. San Francisco, CA"\n },\n {\n "name": "unit",\n "kind": "string",\n "description": "Temperature unit",\n "enumValues": [\n "celsius",\n "fahrenheit"\n ]\n }\n ],\n "bindings": {\n "unit": {\n "input": "preferred_unit"\n }\n }\n }\n ],\n "inputs": [\n {\n "name": "question",\n "kind": "string",\n "default": "What\'s the weather in Paris?"\n },\n {\n "name": "preferred_unit",\n "kind": "string",\n "default": "celsius"\n }\n ],\n "instructions": "system:\\nYou are a helpful assistant with access to tools.\\n\\nuser:\\n{{question}}"\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "tools_mcp_load",\n "description": "Load a prompty with an MCP tool — kind, serverName, connection preserved",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "mcp-test",\n "model": "gpt-4",\n "tools": [\n {\n "name": "filesystem",\n "kind": "mcp",\n "serverName": "fs-server",\n "connection": {\n "kind": "reference",\n "name": "my-mcp"\n }\n }\n ]\n }\n },\n "expected": {\n "tools": [\n {\n "name": "filesystem",\n "kind": "mcp",\n "serverName": "fs-server"\n }\n ]\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "tools_openapi_load",\n "description": "Load a prompty with an OpenAPI tool — specification path preserved",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "openapi-test",\n "model": "gpt-4",\n "tools": [\n {\n "name": "weather_api",\n "kind": "openapi",\n "specification": "./weather.json",\n "connection": {\n "kind": "key",\n "endpoint": "https://api.weather.com"\n }\n }\n ]\n }\n },\n "expected": {\n "tools": [\n {\n "name": "weather_api",\n "kind": "openapi",\n "specification": "./weather.json"\n }\n ]\n },\n "operation": "load"\n }\n },\n {\n "contract": "LoadConformance",\n "operation": "load",\n "params": {\n "source": "unknown"\n },\n "returns": "Agent",\n "vector": {\n "name": "tools_prompty_load",\n "description": "Load a prompty with a PromptyTool — path and mode preserved",\n "stage": "load",\n "input": {\n "frontmatter": {\n "name": "prompty-tool-test",\n "model": "gpt-4",\n "tools": [\n {\n "name": "helper",\n "kind": "prompty",\n "path": "./helper.prompty"\n }\n ]\n }\n },\n "expected": {\n "tools": [\n {\n "name": "helper",\n "kind": "prompty",\n "path": "./helper.prompty"\n }\n ]\n },\n "operation": "load"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "assistant_with_code_block",\n "description": "Code blocks with triple backticks inside a role block are preserved as literal text content",\n "stage": "parse",\n "input": {\n "rendered": "assistant:\\nHere is code:\\n```python\\nprint(\'hello\')\\n```\\n\\nuser:\\nThanks"\n },\n "expected": {\n "messages": [\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "Here is code:\\n```python\\nprint(\'hello\')\\n```"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Thanks"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "consecutive_newlines_between_roles",\n "description": "Extra blank lines between role markers are part of the preceding message\'s content but are stripped by trailing-newline trimming, resulting in clean content",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nHelper\\n\\n\\n\\nuser:\\nHi"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Helper"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "content_trimmed",\n "description": "Leading and trailing newlines are stripped from each message\'s content, but internal whitespace (spaces, indentation) is preserved",\n "stage": "parse",\n "input": {\n "rendered": "system:\\n \\n You are helpful. \\n \\n\\nuser:\\n Hello "\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": " \\n You are helpful. \\n "\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": " Hello "\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "empty_content",\n "description": "A role marker followed immediately by another role marker produces a message with empty string content",\n "stage": "parse",\n "input": {\n "rendered": "system:\\n\\nuser:\\nHello"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": ""\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "markdown_in_content",\n "description": "Markdown formatting (headers, bold, lists) within a role block is preserved verbatim as text content",\n "stage": "parse",\n "input": {\n "rendered": "system:\\n# Title\\n\\nSome **bold** text\\n\\n- item 1\\n- item 2\\n\\nuser:\\nHello"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "# Title\\n\\nSome **bold** text\\n\\n- item 1\\n- item 2"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "multiline_content",\n "description": "Multiple lines within a single role block are preserved as multiline content",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nLine one.\\nLine two.\\nLine three.\\n\\nuser:\\nQuestion"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Line one.\\nLine two.\\nLine three."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Question"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "multiple_assistant_turns",\n "description": "Multiple assistant messages in a conversation",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nYou are helpful.\\n\\nuser:\\nHi\\n\\nassistant:\\nHello!\\n\\nuser:\\nThanks\\n\\nassistant:\\nYou\'re welcome!"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "You are helpful."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "Hello!"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Thanks"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "You\'re welcome!"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "multiple_user_turns",\n "description": "Alternating user and assistant turns produce four messages in conversation order",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nHelper\\n\\nuser:\\nFirst question\\n\\nassistant:\\nFirst answer\\n\\nuser:\\nSecond question"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Helper"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "First question"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "First answer"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Second question"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "no_role_marker_defaults_to_system",\n "description": "Content with no role marker at all defaults to a single system message",\n "stage": "parse",\n "input": {\n "rendered": "Just a plain message with no role marker"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Just a plain message with no role marker"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "role_marker_at_line_start_only",\n "description": "A role name followed by colon mid-line (e.g. \'The user: said\') is NOT treated as a role boundary — only lines matching the boundary regex (role name at line start, colon at end) split messages",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nThe user: said hello\\n\\nuser:\\nHi"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "The user: said hello"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "role_marker_colon_in_content",\n "description": "Colons in content (not at line start) are not treated as role markers",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nHere is a tip: always be kind.\\n\\nuser:\\nTime is 3:30pm"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Here is a tip: always be kind."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Time is 3:30pm"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "single_system",\n "description": "A single system role marker produces one system message",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nYou are helpful."\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "You are helpful."\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "system_user",\n "description": "System and user role markers produce two messages in order",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nYou are helpful.\\n\\nuser:\\nHello"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "You are helpful."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "system_user_assistant",\n "description": "Three role markers (system, user, assistant) produce three messages in order",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nBe concise.\\n\\nuser:\\nHi\\n\\nassistant:\\nHello! How can I help?"\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Be concise."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "Hello! How can I help?"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Parser",\n "operation": "parse",\n "params": {\n "agent": "Agent",\n "rendered": "string",\n "context": "Record?"\n },\n "returns": "Message[]",\n "vector": {\n "name": "thread_nonce_expansion",\n "description": "Thread nonce markers emitted by the renderer are expanded into actual conversation messages by the prepare pipeline. The nonce placeholder is replaced by the thread\'s message list, interleaved at the correct position.",\n "stage": "parse",\n "input": {\n "rendered": "system:\\nYou are helpful.\\n\\n__PROMPTY_THREAD_abcd1234_conversation__\\n\\nuser:\\nHi",\n "thread_inputs": {\n "conversation": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "prev question"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "prev answer"\n }\n ]\n }\n ]\n }\n },\n "expected": {\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "You are helpful."\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "prev question"\n }\n ]\n },\n {\n "role": "assistant",\n "content": [\n {\n "kind": "text",\n "value": "prev answer"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n }\n ]\n },\n "operation": "parse"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_multiple_text_blocks",\n "description": "Anthropic message response with two text content blocks — concatenates them",\n "stage": "process",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "response": {\n "id": "msg_03ZHFVFLIiCCEzvnrwxXqaNO",\n "type": "message",\n "role": "assistant",\n "model": "claude-sonnet-4-20250514",\n "content": [\n {\n "type": "text",\n "text": "Here is the answer:"\n },\n {\n "type": "text",\n "text": " The weather in Paris is sunny."\n }\n ],\n "stop_reason": "end_turn",\n "stop_sequence": null,\n "usage": {\n "input_tokens": 15,\n "output_tokens": 12\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": "Here is the answer: The weather in Paris is sunny."\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_structured_output",\n "description": "Anthropic text response JSON-parsed when has_outputs is true",\n "stage": "process",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "response": {\n "id": "msg-struct",\n "type": "message",\n "role": "assistant",\n "content": [\n {\n "type": "text",\n "text": "{\\"city\\": \\"Paris\\", \\"temp\\": 22}"\n }\n ],\n "model": "claude-sonnet-4-20250514",\n "stop_reason": "end_turn",\n "stop_sequence": null,\n "usage": {\n "input_tokens": 20,\n "output_tokens": 10\n }\n },\n "has_outputs": true\n },\n "expected": {\n "result": {\n "city": "Paris",\n "temp": 22\n }\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_text",\n "description": "Anthropic message response with a single text content block — extracts the text",\n "stage": "process",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "response": {\n "id": "msg_01XFDUDYJgAACzvnptvVoYEL",\n "type": "message",\n "role": "assistant",\n "model": "claude-sonnet-4-20250514",\n "content": [\n {\n "type": "text",\n "text": "Hello!"\n }\n ],\n "stop_reason": "end_turn",\n "stop_sequence": null,\n "usage": {\n "input_tokens": 10,\n "output_tokens": 3\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": "Hello!"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_tool_use",\n "description": "Anthropic message response with tool_use content block — extracts ToolCall with JSON-stringified input",\n "stage": "process",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "response": {\n "id": "msg_02YGEUEKHgBBDzvnquwWpZMN",\n "type": "message",\n "role": "assistant",\n "model": "claude-sonnet-4-20250514",\n "content": [\n {\n "type": "tool_use",\n "id": "toolu_1",\n "name": "get_weather",\n "input": {\n "city": "Paris"\n }\n }\n ],\n "stop_reason": "tool_use",\n "stop_sequence": null,\n "usage": {\n "input_tokens": 25,\n "output_tokens": 20\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n {\n "id": "toolu_1",\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n }\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_empty_content",\n "description": "Chat completion with empty string content returns empty string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-empty",\n "object": "chat.completion",\n "created": 1700000000,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "",\n "refusal": null\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 5,\n "completion_tokens": 0,\n "total_tokens": 5\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": ""\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_multiple_tool_calls",\n "description": "Chat completion with two parallel tool calls — extracts both ToolCall objects",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-tool2",\n "object": "chat.completion",\n "created": 1700000002,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_a",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n }\n },\n {\n "id": "call_b",\n "type": "function",\n "function": {\n "name": "get_time",\n "arguments": "{\\"timezone\\":\\"Europe/Paris\\"}"\n }\n }\n ],\n "refusal": null\n },\n "finish_reason": "tool_calls"\n }\n ],\n "usage": {\n "prompt_tokens": 30,\n "completion_tokens": 28,\n "total_tokens": 58\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n {\n "id": "call_a",\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n },\n {\n "id": "call_b",\n "name": "get_time",\n "arguments": "{\\"timezone\\":\\"Europe/Paris\\"}"\n }\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_null_content",\n "description": "Chat completion with null content and no tool calls — returns empty string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-null1",\n "object": "chat.completion",\n "created": 1700000006,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "refusal": null\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 10,\n "completion_tokens": 0,\n "total_tokens": 10\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": ""\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_refusal",\n "description": "Chat completion where the model refused the request — returns the refusal string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-refusal1",\n "object": "chat.completion",\n "created": 1700000005,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "refusal": "I cannot help with that"\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 15,\n "completion_tokens": 8,\n "total_tokens": 23\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": "I cannot help with that"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_structured_invalid_json",\n "description": "Chat completion with has_outputs=true but content is not valid JSON — falls back to raw string gracefully",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-struct2",\n "object": "chat.completion",\n "created": 1700000004,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "not valid json",\n "refusal": null\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 20,\n "completion_tokens": 5,\n "total_tokens": 25\n }\n },\n "has_outputs": true\n },\n "expected": {\n "result": "not valid json"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_structured_output",\n "description": "Chat completion with structured output (has_outputs=true) — JSON-parses the content string into an object",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-struct1",\n "object": "chat.completion",\n "created": 1700000003,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "{\\"city\\":\\"Paris\\",\\"temp\\":72}",\n "refusal": null\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 20,\n "completion_tokens": 12,\n "total_tokens": 32\n }\n },\n "has_outputs": true\n },\n "expected": {\n "result": {\n "city": "Paris",\n "temp": 72\n }\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_text_content",\n "description": "Basic chat completion with text content — extracts the message string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-abc123",\n "object": "chat.completion",\n "created": 1700000000,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Hello!",\n "refusal": null\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 10,\n "completion_tokens": 2,\n "total_tokens": 12\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": "Hello!"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_tool_calls",\n "description": "Chat completion with a single tool call — extracts ToolCall list",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "response": {\n "id": "chatcmpl-tool1",\n "object": "chat.completion",\n "created": 1700000001,\n "model": "gpt-4o-mini",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_1",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n }\n }\n ],\n "refusal": null\n },\n "finish_reason": "tool_calls"\n }\n ],\n "usage": {\n "prompt_tokens": 25,\n "completion_tokens": 15,\n "total_tokens": 40\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n {\n "id": "call_1",\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n }\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "embedding_batch",\n "description": "Batch embedding response with 3 inputs — returns list of 3 vectors",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "embedding",\n "response": {\n "object": "list",\n "data": [\n {\n "object": "embedding",\n "index": 0,\n "embedding": [\n 0.1,\n 0.2\n ]\n },\n {\n "object": "embedding",\n "index": 1,\n "embedding": [\n 0.3,\n 0.4\n ]\n },\n {\n "object": "embedding",\n "index": 2,\n "embedding": [\n 0.5,\n 0.6\n ]\n }\n ],\n "model": "text-embedding-3-small",\n "usage": {\n "prompt_tokens": 15,\n "total_tokens": 15\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n [\n 0.1,\n 0.2\n ],\n [\n 0.3,\n 0.4\n ],\n [\n 0.5,\n 0.6\n ]\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "embedding_single",\n "description": "Single embedding response — extracts the vector directly",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "embedding",\n "response": {\n "object": "list",\n "data": [\n {\n "object": "embedding",\n "index": 0,\n "embedding": [\n 0.1,\n 0.2,\n 0.3\n ]\n }\n ],\n "model": "text-embedding-3-small",\n "usage": {\n "prompt_tokens": 5,\n "total_tokens": 5\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n 0.1,\n 0.2,\n 0.3\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "image_b64",\n "description": "Image generation response with base64 data — extracts the b64_json string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "image",\n "response": {\n "created": 1700000011,\n "data": [\n {\n "b64_json": "base64data",\n "url": null,\n "revised_prompt": null\n }\n ]\n },\n "has_outputs": false\n },\n "expected": {\n "result": "base64data"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "image_revised_prompt",\n "description": "Image generation response with revised_prompt — primary result is the URL, revised_prompt is metadata",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "image",\n "response": {\n "created": 1700000012,\n "data": [\n {\n "url": "https://img.example.com/cat.png",\n "revised_prompt": "A cute cat sitting on a windowsill in warm sunlight"\n }\n ]\n },\n "has_outputs": false\n },\n "expected": {\n "result": "https://img.example.com/cat.png"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "image_url",\n "description": "Image generation response with URL — extracts the image URL",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "image",\n "response": {\n "created": 1700000010,\n "data": [\n {\n "url": "https://img.example.com/generated.png",\n "revised_prompt": null\n }\n ]\n },\n "has_outputs": false\n },\n "expected": {\n "result": "https://img.example.com/generated.png"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_empty_output",\n "description": "Responses API with output_text empty returns empty string",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "response": {\n "id": "resp-empty",\n "object": "response",\n "status": "completed",\n "output": [],\n "output_text": "",\n "model": "gpt-4o"\n },\n "has_outputs": false\n },\n "expected": {\n "result": ""\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_structured",\n "description": "Responses API with structured output (has_outputs=true) — JSON-parses output_text into object",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "response": {\n "id": "resp_struct1",\n "object": "response",\n "created_at": 1700000022,\n "model": "gpt-4o-mini",\n "status": "completed",\n "output": [\n {\n "type": "message",\n "id": "msg_002",\n "status": "completed",\n "role": "assistant",\n "content": [\n {\n "type": "output_text",\n "text": "{\\"city\\":\\"Paris\\"}",\n "annotations": []\n }\n ]\n }\n ],\n "output_text": "{\\"city\\":\\"Paris\\"}",\n "usage": {\n "input_tokens": 20,\n "output_tokens": 8,\n "total_tokens": 28\n }\n },\n "has_outputs": true\n },\n "expected": {\n "result": {\n "city": "Paris"\n }\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_text",\n "description": "Responses API with plain text output — extracts output_text",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "response": {\n "id": "resp_abc123",\n "object": "response",\n "created_at": 1700000020,\n "model": "gpt-4o-mini",\n "status": "completed",\n "output": [\n {\n "type": "message",\n "id": "msg_001",\n "status": "completed",\n "role": "assistant",\n "content": [\n {\n "type": "output_text",\n "text": "Hello!",\n "annotations": []\n }\n ]\n }\n ],\n "output_text": "Hello!",\n "usage": {\n "input_tokens": 10,\n "output_tokens": 2,\n "total_tokens": 12\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": "Hello!"\n },\n "operation": "process"\n }\n },\n {\n "contract": "Processor",\n "operation": "process",\n "params": {\n "agent": "Agent",\n "response": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_tool_calls",\n "description": "Responses API with function_call output items — extracts ToolCall list with call_id preserved",\n "stage": "process",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "response": {\n "id": "resp_tool1",\n "object": "response",\n "created_at": 1700000021,\n "model": "gpt-4o-mini",\n "status": "completed",\n "output": [\n {\n "type": "function_call",\n "id": "fc_001",\n "call_id": "call_1",\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}",\n "status": "completed"\n }\n ],\n "output_text": "",\n "usage": {\n "input_tokens": 25,\n "output_tokens": 18,\n "total_tokens": 43\n }\n },\n "has_outputs": false\n },\n "expected": {\n "result": [\n {\n "id": "call_1",\n "name": "get_weather",\n "arguments": "{\\"city\\":\\"Paris\\"}"\n }\n ]\n },\n "operation": "process"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "conditional_block",\n "description": "Jinja2 if/else block where condition is true",\n "stage": "render",\n "input": {\n "template": "{% if premium %}Welcome VIP!{% else %}Hello!{% endif %}",\n "engine": "jinja2",\n "inputs": {\n "premium": true\n }\n },\n "expected": {\n "rendered": "Welcome VIP!"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "conditional_false",\n "description": "Jinja2 if/else block where condition is false",\n "stage": "render",\n "input": {\n "template": "{% if premium %}Welcome VIP!{% else %}Hello!{% endif %}",\n "engine": "jinja2",\n "inputs": {\n "premium": false\n }\n },\n "expected": {\n "rendered": "Hello!"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "default_filter",\n "description": "Jinja2 default filter provides fallback for missing variables",\n "stage": "render",\n "input": {\n "template": "{{name|default(\\"stranger\\")}}",\n "engine": "jinja2",\n "inputs": {}\n },\n "expected": {\n "rendered": "stranger"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_basic",\n "description": "Jinja2 built-in filter (upper) works in prompty templates",\n "stage": "render",\n "input": {\n "template": "{{name|upper}}",\n "engine": "jinja2",\n "inputs": {\n "name": "hello"\n }\n },\n "expected": {\n "rendered": "HELLO"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_join",\n "description": "Jinja2 join filter joins a list with separator",\n "stage": "render",\n "input": {\n "template": "{{ items | join(\', \') }}",\n "engine": "jinja2",\n "inputs": {\n "items": [\n "a",\n "b",\n "c"\n ]\n }\n },\n "expected": {\n "rendered": "a, b, c"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_length",\n "description": "Jinja2 length filter returns collection size",\n "stage": "render",\n "input": {\n "template": "{{ items | length }}",\n "engine": "jinja2",\n "inputs": {\n "items": [\n 1,\n 2,\n 3,\n 4,\n 5\n ]\n }\n },\n "expected": {\n "rendered": "5"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_lower",\n "description": "Jinja2 lower filter converts to lowercase",\n "stage": "render",\n "input": {\n "template": "{{ name | lower }}",\n "engine": "jinja2",\n "inputs": {\n "name": "HELLO"\n }\n },\n "expected": {\n "rendered": "hello"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_trim",\n "description": "Jinja2 trim filter removes surrounding whitespace",\n "stage": "render",\n "input": {\n "template": "{{ name | trim }}",\n "engine": "jinja2",\n "inputs": {\n "name": " hello "\n }\n },\n "expected": {\n "rendered": "hello"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "filter_upper",\n "description": "Jinja2 upper filter converts to uppercase",\n "stage": "render",\n "input": {\n "template": "{{ name | upper }}",\n "engine": "jinja2",\n "inputs": {\n "name": "hello"\n }\n },\n "expected": {\n "rendered": "HELLO"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "for_loop",\n "description": "Jinja2 for loop iterating over a list of items",\n "stage": "render",\n "input": {\n "template": "Items: {% for item in items %}{{item}} {% endfor %}",\n "engine": "jinja2",\n "inputs": {\n "items": [\n "a",\n "b",\n "c"\n ]\n }\n },\n "expected": {\n "rendered": "Items: a b c "\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "html_not_escaped",\n "description": "HTML content is NOT escaped — prompty templates are not HTML templates",\n "stage": "render",\n "input": {\n "template": "{{content}}",\n "engine": "jinja2",\n "inputs": {\n "content": "bold"\n }\n },\n "expected": {\n "rendered": "bold"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "jinja2_comment",\n "description": "Jinja2 comments are stripped from output",\n "stage": "render",\n "input": {\n "template": "Hello{# this is a comment #} World",\n "engine": "jinja2",\n "inputs": {}\n },\n "expected": {\n "rendered": "Hello World"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "missing_variable_renders_empty",\n "description": "Undefined Jinja2 variable renders as empty string with default Undefined config",\n "stage": "render",\n "input": {\n "template": "Hello {{name}}!",\n "engine": "jinja2",\n "inputs": {}\n },\n "expected": {\n "rendered": "Hello !"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "multiple_variables",\n "description": "Multiple Jinja2 variables of different types including an integer",\n "stage": "render",\n "input": {\n "template": "{{first}} {{last}} is {{age}}",\n "engine": "jinja2",\n "inputs": {\n "first": "Jane",\n "last": "Doe",\n "age": 30\n }\n },\n "expected": {\n "rendered": "Jane Doe is 30"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "mustache_inverted",\n "description": "Mustache inverted section renders when value is falsy",\n "stage": "render",\n "input": {\n "template": "{{^show}}Hidden{{/show}}",\n "engine": "mustache",\n "inputs": {\n "show": false\n }\n },\n "expected": {\n "rendered": "Hidden"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "mustache_loop",\n "description": "Mustache section iterates over list items",\n "stage": "render",\n "input": {\n "template": "{{#items}}{{.}} {{/items}}",\n "engine": "mustache",\n "inputs": {\n "items": [\n "a",\n "b",\n "c"\n ]\n }\n },\n "expected": {\n "rendered": "a b c "\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "mustache_section",\n "description": "Mustache truthy section renders its content",\n "stage": "render",\n "input": {\n "template": "{{#show}}Visible{{/show}}",\n "engine": "mustache",\n "inputs": {\n "show": true\n }\n },\n "expected": {\n "rendered": "Visible"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "mustache_simple",\n "description": "Basic Mustache variable substitution",\n "stage": "render",\n "input": {\n "template": "Hello {{name}}!",\n "engine": "mustache",\n "inputs": {\n "name": "World"\n }\n },\n "expected": {\n "rendered": "Hello World!"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "nested_object",\n "description": "Jinja2 dot notation access into a nested object",\n "stage": "render",\n "input": {\n "template": "{{user.name}} ({{user.email}})",\n "engine": "jinja2",\n "inputs": {\n "user": {\n "name": "Jane",\n "email": "jane@test.com"\n }\n }\n },\n "expected": {\n "rendered": "Jane (jane@test.com)"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "role_markers_preserved",\n "description": "Role markers (system:/user:) are passed through by the renderer — parsing is the parser\'s job",\n "stage": "render",\n "input": {\n "template": "system:\\nYou are helpful.\\n\\nuser:\\n{{question}}",\n "engine": "jinja2",\n "inputs": {\n "question": "Hello"\n }\n },\n "expected": {\n "rendered": "system:\\nYou are helpful.\\n\\nuser:\\nHello"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "simple_substitution",\n "description": "Basic Jinja2 variable substitution with a single variable",\n "stage": "render",\n "input": {\n "template": "Hello {{name}}!",\n "engine": "jinja2",\n "inputs": {\n "name": "World"\n }\n },\n "expected": {\n "rendered": "Hello World!"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "thread_nonce_injection",\n "description": "Thread-kind input is replaced with a nonce marker instead of being rendered directly",\n "stage": "render",\n "input": {\n "template": "system:\\nYou are helpful.\\n\\n{{conversation}}\\n\\nuser:\\n{{question}}",\n "engine": "jinja2",\n "inputs": {\n "question": "Hi",\n "conversation": {\n "_kind": "thread",\n "messages": [\n {\n "role": "user",\n "content": "previous question"\n },\n {\n "role": "assistant",\n "content": "previous answer"\n }\n ]\n }\n }\n },\n "expected": {\n "nonce_pattern": "^system:\\nYou are helpful\\\\.\\n\\n__PROMPTY_THREAD_[a-f0-9]{8}_conversation__\\n\\nuser:\\nHi$"\n },\n "operation": "render"\n }\n },\n {\n "contract": "Renderer",\n "operation": "render",\n "params": {\n "agent": "Agent",\n "template": "string",\n "inputs": "Record"\n },\n "returns": "string",\n "vector": {\n "name": "whitespace_preserved",\n "description": "Whitespace including indentation and newlines is exactly preserved",\n "stage": "render",\n "input": {\n "template": "line1\\n line2\\n line3",\n "engine": "jinja2",\n "inputs": {}\n },\n "expected": {\n "rendered": "line1\\n line2\\n line3"\n },\n "operation": "render"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "replay",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "max_iterations",\n "stage": "replay",\n "input": {\n "clock": "2026-06-28T00:00:00Z",\n "sessionId": "session-1",\n "turnId": "turn-1",\n "maxIterations": 1\n },\n "expected": [\n "session:session_start:session-1:turn-1",\n "turn:turn_start:0",\n "turn:llm_start:0",\n "turn:llm_complete:0",\n "session:checkpoint_created:session-1:turn-1",\n "turn:permission_requested:0:exec-1-permission",\n "turn:permission_completed:0:true",\n "turn:tool_execution_start:0:add",\n "turn:tool_execution_complete:0:add:true",\n "turn:tool_result:0:add:true",\n "turn:messages_updated:0",\n "turn:error:1:max_iterations",\n "turn:turn_end:1:error",\n "session:session_end:session-1:turn-1:error",\n "summary:session-1:error:turns=1:checkpoints=1"\n ],\n "operation": "replay"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "replay",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "no_tool",\n "stage": "replay",\n "input": {\n "clock": "2026-06-28T00:00:00Z",\n "sessionId": "session-1",\n "turnId": "turn-1",\n "inputs": {\n "name": "Ada"\n },\n "maxIterations": 3\n },\n "expected": [\n "session:session_start:session-1:turn-1",\n "turn:turn_start:0",\n "turn:llm_start:0",\n "turn:llm_complete:0",\n "session:checkpoint_created:session-1:turn-1",\n "turn:turn_end:1:success",\n "session:session_end:session-1:turn-1:success",\n "summary:session-1:success:turns=1:checkpoints=1"\n ],\n "operation": "replay"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "replay",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "permission_denied",\n "stage": "replay",\n "input": {\n "clock": "2026-06-28T00:00:00Z",\n "sessionId": "session-1",\n "turnId": "turn-1"\n },\n "expected": [\n "session:session_start:session-1:turn-1",\n "turn:turn_start:0",\n "turn:llm_start:0",\n "turn:llm_complete:0",\n "session:checkpoint_created:session-1:turn-1",\n "turn:permission_requested:0:exec-1-permission",\n "turn:permission_completed:0:false",\n "turn:tool_result:0:add:false:permission_denied",\n "turn:messages_updated:0",\n "turn:llm_start:1",\n "turn:llm_complete:1",\n "session:checkpoint_created:session-1:turn-1",\n "turn:turn_end:2:success",\n "session:session_end:session-1:turn-1:success",\n "summary:session-1:success:turns=1:checkpoints=2"\n ],\n "operation": "replay"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "replay",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "tool_failure",\n "stage": "replay",\n "input": {\n "clock": "2026-06-28T00:00:00Z",\n "sessionId": "session-1",\n "turnId": "turn-1"\n },\n "expected": [\n "session:session_start:session-1:turn-1",\n "turn:turn_start:0",\n "turn:llm_start:0",\n "turn:llm_complete:0",\n "session:checkpoint_created:session-1:turn-1",\n "turn:permission_requested:0:exec-1-permission",\n "turn:permission_completed:0:true",\n "turn:tool_execution_start:0:fail",\n "turn:tool_execution_complete:0:fail:false:exception",\n "turn:tool_result:0:fail:false:exception",\n "turn:messages_updated:0",\n "turn:llm_start:1",\n "turn:llm_complete:1",\n "session:checkpoint_created:session-1:turn-1",\n "turn:turn_end:2:success",\n "session:session_end:session-1:turn-1:success",\n "summary:session-1:success:turns=1:checkpoints=2"\n ],\n "operation": "replay"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "replay",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "tool_success",\n "stage": "replay",\n "input": {\n "clock": "2026-06-28T00:00:00Z",\n "sessionId": "session-1",\n "turnId": "turn-1"\n },\n "expected": [\n "session:session_start:session-1:turn-1",\n "turn:turn_start:0",\n "turn:llm_start:0",\n "turn:llm_complete:0",\n "session:checkpoint_created:session-1:turn-1",\n "turn:permission_requested:0:exec-1-permission",\n "turn:permission_completed:0:true",\n "turn:tool_execution_start:0:add",\n "turn:tool_execution_complete:0:add:true",\n "turn:tool_result:0:add:true",\n "turn:messages_updated:0",\n "turn:llm_start:1",\n "turn:llm_complete:1",\n "session:checkpoint_created:session-1:turn-1",\n "turn:turn_end:2:success",\n "session:session_end:session-1:turn-1:success",\n "summary:session-1:success:turns=1:checkpoints=2"\n ],\n "operation": "replay"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "assistant_tool_calls_metadata",\n "description": "When the LLM returns tool calls, the assistant message appended to the conversation must have role=\'assistant\', empty content, and metadata.tool_calls containing the full tool_calls array with id/type/function for each call.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Compare weather in two cities."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-meta-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_meta_paris",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_meta_london",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_meta_paris",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_meta_london",\n "name": "get_weather",\n "arguments": {\n "city": "London"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_meta_paris",\n "result": "72°F sunny"\n },\n {\n "tool_call_id": "call_meta_london",\n "result": "59°F cloudy"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-meta-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Paris is warmer and sunnier than London today.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "Paris is warmer and sunnier than London today.",\n "iterations": 2,\n "assistant_tool_calls_message": {\n "role": "assistant",\n "content": "",\n "metadata": {\n "tool_calls": [\n {\n "id": "call_meta_paris",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_meta_london",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n }\n }\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "async_tool_function",\n "description": "Agent loop correctly handles async tool functions",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are helpful."\n },\n {\n "role": "user",\n "content": "Look up the data"\n }\n ],\n "tools": [\n {\n "name": "lookup",\n "kind": "function",\n "description": "Look up data",\n "parameters": [\n {\n "name": "query",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "lookup": "returns lookup result"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-async-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_async1",\n "type": "function",\n "function": {\n "name": "lookup",\n "arguments": "{\\"query\\":\\"test\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_async1",\n "name": "lookup",\n "arguments": {\n "query": "test"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_async1",\n "result": "found: test data"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-async-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "I found: test data",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "I found: test data",\n "iterations": 2,\n "total_messages": 6\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "bindings_injected",\n "description": "Tool has a binding {unit: {input: \'preferred_unit\'}}. The LLM omits the bound parameter. At execution time, the binding injects the value from the parent inputs, so the tool receives the merged arguments.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "parent_inputs": {\n "preferred_unit": "celsius"\n },\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n },\n {\n "name": "unit",\n "kind": "string",\n "required": false\n }\n ],\n "bindings": {\n "unit": {\n "input": "preferred_unit"\n }\n }\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-binding-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_bind_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_bind_001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "expected_execution_args": {\n "get_weather": {\n "city": "Paris",\n "unit": "celsius"\n }\n },\n "tool_results": [\n {\n "tool_call_id": "call_bind_001",\n "result": "22°C sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-binding-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 22°C and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 22°C and sunny.",\n "iterations": 2,\n "total_messages": 6,\n "notes": "The binding {unit: {input: \'preferred_unit\'}} resolves \'preferred_unit\' from parent_inputs to \'celsius\' and injects it into the tool call arguments before execution, overriding any LLM-provided value."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "cancellation_before_llm",\n "description": "§13.2 Cancellation — Cancel token is already cancelled before the first LLM call. No LLM calls are made and CancelledError is raised.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string"\n },\n "cancel": {\n "cancelled_at": "before_iteration"\n },\n "on_event": true\n },\n "sequence": [],\n "expected": {\n "error": "CancelledError",\n "iterations": 0,\n "total_messages": 2,\n "events": [\n {\n "type": "cancelled",\n "data": {\n "reason": "Cancellation requested before first iteration"\n }\n }\n ],\n "notes": "No LLM calls are made. The cancel token is checked at the top of each iteration. Since it is already cancelled, the loop exits immediately with CancelledError."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "cancellation_between_iterations",\n "description": "§13.2 Cancellation — Turn 1 completes (tool call + result). Cancel fires before turn 2. The loop exits with CancelledError after 1 iteration.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "cancel": {\n "cancelled_at": "before_iteration_2"\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-cancel-iter-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_cancel_iter001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_cancel_iter001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_cancel_iter001",\n "result": "72°F sunny"\n }\n ]\n }\n ],\n "expected": {\n "error": "CancelledError",\n "iterations": 1,\n "total_messages": 5,\n "events": [\n {\n "type": "status",\n "data": {\n "message": "Starting agent loop"\n }\n },\n {\n "type": "tool_call_start",\n "data": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "type": "tool_result",\n "data": {\n "name": "get_weather",\n "result": "72°F sunny"\n }\n },\n {\n "type": "messages_updated",\n "data": {\n "message_count": 5\n }\n },\n {\n "type": "cancelled",\n "data": {\n "reason": "Cancellation requested before iteration 2"\n }\n }\n ],\n "notes": "Iteration 1 runs to completion (LLM call + tool execution + message append). The cancel token is checked at the top of iteration 2, causing CancelledError. The 5 messages are: system, user, assistant(tool_calls), tool(result), but the second LLM call never happens."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "cancellation_between_tools",\n "description": "§13.2 Cancellation — LLM requests 2 tool calls. Cancel fires after the first tool executes. The second tool must NOT be called.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Compare weather in Paris and London."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "cancel": {\n "cancelled_at": "after_tool_0"\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-cancel-tools-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_cancel_paris",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_cancel_london",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_cancel_paris",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_cancel_london",\n "name": "get_weather",\n "arguments": {\n "city": "London"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_cancel_paris",\n "result": "72°F sunny"\n }\n ]\n }\n ],\n "expected": {\n "error": "CancelledError",\n "iterations": 1,\n "tools_executed": 1,\n "events": [\n {\n "type": "status",\n "data": {\n "message": "Starting agent loop"\n }\n },\n {\n "type": "tool_call_start",\n "data": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "type": "tool_result",\n "data": {\n "name": "get_weather",\n "result": "72°F sunny"\n }\n },\n {\n "type": "cancelled",\n "data": {\n "reason": "Cancellation requested after tool execution"\n }\n }\n ],\n "notes": "Cancel is checked between tool executions. The first tool (Paris) completes, then the cancel token fires. The second tool (London) must NOT be executed. Only one tool_result event is emitted."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "context_no_trim_when_fits",\n "description": "§13.3 Context Window — Messages fit within the context budget. No trimming occurs and all messages are preserved.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is 2 + 2?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n },\n "context_budget": 10000\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-ctx-nofit-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "2 + 2 equals 4.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "2 + 2 equals 4.",\n "iterations": 1,\n "total_messages": 3,\n "trimmed_messages": null,\n "notes": "When messages fit within the context_budget (10000 characters), no trimming occurs. The trimmed_messages field is null to indicate no modification was needed."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "context_preserves_system_messages",\n "description": "§13.3 Context Window — Two system messages plus many user/assistant pairs. After trimming, both system messages MUST still be present.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "system",\n "content": "Always respond in a formal tone."\n },\n {\n "role": "user",\n "content": "Tell me about ancient Egypt and all the pharaohs and their dynasties in great detail."\n },\n {\n "role": "assistant",\n "content": "Ancient Egypt\'s history spans over 3000 years. The Old Kingdom saw the building of the Great Pyramids at Giza under pharaohs Khufu, Khafre, and Menkaure. The Middle Kingdom was a period of reunification. The New Kingdom included famous rulers like Hatshepsut, Thutmose III, Akhenaten, and Tutankhamun."\n },\n {\n "role": "user",\n "content": "Now tell me about Greek philosophy covering all major philosophers and their contributions."\n },\n {\n "role": "assistant",\n "content": "Greek philosophy began with the Pre-Socratics. Socrates developed the Socratic method. Plato founded the Academy and wrote The Republic. Aristotle studied under Plato and founded the Lyceum, contributing to logic, metaphysics, ethics, and natural sciences. The Stoics and Epicureans followed in the Hellenistic period."\n },\n {\n "role": "user",\n "content": "What is the weather today?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string"\n },\n "context_budget": 300\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-ctx-sys-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "I would need to know your city to provide the weather.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "I would need to know your city to provide the weather.",\n "iterations": 1,\n "trimmed_messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "system",\n "content": "Always respond in a formal tone."\n },\n {\n "role": "system",\n "content": "[Summary of earlier conversation] User asked about ancient Egypt and pharaohs, then about Greek philosophy and major philosophers."\n },\n {\n "role": "user",\n "content": "What is the weather today?"\n }\n ],\n "notes": "ALL system messages are preserved during context trimming — they are never dropped regardless of budget. A summary is inserted after the last system message. The most recent user message is always kept."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "context_trim_basic",\n "description": "§13.3 Context Window — Messages exceed the context budget. Oldest user/assistant pairs are dropped and a summary is inserted after the system message.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Tell me about the history of Rome. I want a very detailed answer covering all major periods."\n },\n {\n "role": "assistant",\n "content": "Rome was founded in 753 BC. The Roman Kingdom lasted until 509 BC when the Republic was established. The Republic expanded through the Punic Wars and conquered the Mediterranean. Julius Caesar rose to power and was assassinated in 44 BC. Augustus became the first Emperor in 27 BC, beginning the Roman Empire."\n },\n {\n "role": "user",\n "content": "Now tell me about the fall of Rome. Again, very detailed with all the key events and dates."\n },\n {\n "role": "assistant",\n "content": "The decline of Rome began in the 3rd century with the Crisis of the Third Century. Diocletian split the empire in 285 AD. Constantine founded Constantinople in 330 AD. The Western Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. The Eastern Empire continued as the Byzantine Empire until 1453."\n },\n {\n "role": "user",\n "content": "Tell me about medieval Europe after Rome fell. Cover the major kingdoms and events."\n },\n {\n "role": "assistant",\n "content": "After Rome fell, Europe entered the Early Middle Ages. The Franks under Clovis established a kingdom in Gaul. Charlemagne united much of Western Europe and was crowned Emperor in 800 AD. The Viking Age began in 793 AD. The feudal system developed across Europe. The High Middle Ages saw the Crusades beginning in 1095."\n },\n {\n "role": "user",\n "content": "Now tell me about the Renaissance period."\n },\n {\n "role": "assistant",\n "content": "The Renaissance began in Italy in the 14th century. Florence was the center under the Medici family. Key figures included Leonardo da Vinci, Michelangelo, and Raphael. The printing press was invented by Gutenberg around 1440. The Renaissance spread north to France, England, and the Low Countries. It marked a transition from medieval to modern thinking."\n },\n {\n "role": "user",\n "content": "Finally, what is the weather in Paris today?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string"\n },\n "context_budget": 500\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-ctx-trim-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_ctx_w001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_ctx_w001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_ctx_w001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-ctx-trim-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 72°F and sunny.",\n "iterations": 2,\n "trimmed_messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "system",\n "content": "[Summary of earlier conversation] User asked about the history of Rome, the fall of Rome, medieval Europe, and the Renaissance period."\n },\n {\n "role": "user",\n "content": "Finally, what is the weather in Paris today?"\n }\n ],\n "summary_contains": "User asked",\n "notes": "Context trimming drops oldest user/assistant pairs to fit within the 500-character budget. The system message is always preserved. A summary message (role=system) is injected after the original system message. The most recent user message is always kept."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "empty_tool_result",\n "description": "Tool function returns an empty string. The tool result message must still be sent to the LLM with empty content — it must not be skipped or filtered out.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Clear the cache."\n }\n ],\n "tools": [\n {\n "name": "clear_cache",\n "kind": "function",\n "description": "Clear the application cache, returns empty on success",\n "parameters": []\n }\n ],\n "tool_functions": {\n "clear_cache": "returns empty string on success"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-empty-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_empty_001",\n "type": "function",\n "function": {\n "name": "clear_cache",\n "arguments": "{}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_empty_001",\n "name": "clear_cache",\n "arguments": {}\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_empty_001",\n "result": ""\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-empty-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The cache has been cleared successfully.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The cache has been cleared successfully.",\n "iterations": 2,\n "total_messages": 6,\n "tool_result_message": {\n "role": "tool",\n "content": [\n {\n "type": "text",\n "text": ""\n }\n ],\n "metadata": {\n "tool_call_id": "call_empty_001"\n }\n },\n "notes": "Empty string result must NOT be skipped. The tool result message is still appended to the conversation so the LLM can see the tool was executed successfully."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "events_basic_tool_loop",\n "description": "§13.1 Events — A 2-turn tool-call loop emits the full event lifecycle: status → tool_call_start → tool_result → messages_updated → done.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-evt-basic-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_evt_w001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_evt_w001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_evt_w001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-evt-basic-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 72°F and sunny.",\n "iterations": 2,\n "total_messages": 6,\n "events": [\n {\n "type": "status",\n "data": {\n "message": "Starting agent loop"\n }\n },\n {\n "type": "tool_call_start",\n "data": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "type": "tool_result",\n "data": {\n "name": "get_weather",\n "result": "72°F sunny"\n }\n },\n {\n "type": "messages_updated",\n "data": {\n "message_count": 5\n }\n },\n {\n "type": "done",\n "data": {\n "response": "The weather in Paris is 72°F and sunny."\n }\n }\n ],\n "notes": "Events must be emitted in this exact order. tool_call_start fires before tool execution; tool_result fires after. messages_updated fires when tool result messages are appended. done is always the last event."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "events_error_logged",\n "description": "§13.1 Events — A tool function raises an exception during execution. The exception is caught and returned as an error string to the LLM. The loop continues normally.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "raises RuntimeError"\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-evt-err-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_evt_err001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_evt_err001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_evt_err001",\n "result": "Error calling \'get_weather\': RuntimeError: Weather service unavailable"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-evt-err-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Sorry, the weather service is currently unavailable.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "Sorry, the weather service is currently unavailable.",\n "iterations": 2,\n "total_messages": 6,\n "events": [\n {\n "type": "tool_call_start"\n },\n {\n "type": "tool_result"\n },\n {\n "type": "messages_updated"\n },\n {\n "type": "done"\n }\n ],\n "notes": "Tool exceptions are caught by the dispatcher and returned as error strings. The loop continues — this is a success path. Events verify type sequence only (payloads vary by runtime)."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "events_no_tools",\n "description": "§13.1 Events — Single-turn completion with no tool calls. Only the done event is emitted.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is 2 + 2?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-evt-notool-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "2 + 2 equals 4.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "2 + 2 equals 4.",\n "iterations": 1,\n "total_messages": 3,\n "events": [\n {\n "type": "done",\n "data": {\n "response": "2 + 2 equals 4."\n }\n }\n ],\n "notes": "When the LLM returns content without tool calls, no tool_call_start/tool_result/messages_updated events fire. Only the done event is emitted."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "guardrail_all_pass",\n "description": "§13.4 Guardrails — All guardrails (input, output, tool) are configured but all pass. The agent loop completes normally.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "guardrails": {\n "input": {\n "action": "allow"\n },\n "output": {\n "action": "allow"\n },\n "tool": {\n "deny_tools": []\n }\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-guard-pass-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_guard_pass001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_guard_pass001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_guard_pass001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-guard-pass-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 72°F and sunny.",\n "iterations": 2,\n "total_messages": 6,\n "denied_tools": [],\n "notes": "All three guardrail types are configured but none deny. Input guardrail allows, tool guardrail has an empty deny list, and output guardrail allows. The agent loop completes as if no guardrails were present."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "guardrail_input_deny",\n "description": "§13.4 Guardrails — Input guardrail denies the request before any LLM call is made. Returns GuardrailError with the denial reason.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "My SSN is 123-45-6789, can you look that up?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n },\n "guardrails": {\n "input": {\n "action": "deny",\n "reason": "Contains PII"\n }\n }\n },\n "sequence": [],\n "expected": {\n "error": "GuardrailError",\n "error_reason": "Contains PII",\n "iterations": 0,\n "total_messages": 2,\n "notes": "Input guardrail runs before the first LLM call. When it returns action=deny, the loop immediately raises GuardrailError with the provided reason. No LLM calls or tool executions occur."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "guardrail_output_deny",\n "description": "§13.4 Guardrails — Input guardrail passes, LLM returns a response, but the output guardrail denies it. Returns GuardrailError.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Tell me something."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n },\n "guardrails": {\n "input": {\n "action": "allow"\n },\n "output": {\n "action": "deny",\n "reason": "Response contains harmful content"\n }\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-guard-out-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Here is some problematic content that the guardrail catches.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "error": "GuardrailError",\n "error_reason": "Response contains harmful content",\n "iterations": 1,\n "notes": "The input guardrail passes (action=allow). The LLM returns a response. The output guardrail runs on the response and returns action=deny, causing GuardrailError. The LLM was called once but the result is not returned to the user."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "guardrail_tool_deny",\n "description": "§13.4 Guardrails — LLM requests 2 tool calls. Tool guardrail denies one (dangerous_tool) but allows the other (get_weather). Denied tool gets a synthetic error result.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Get the weather and also run the dangerous operation."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "dangerous_tool",\n "kind": "function",\n "description": "A dangerous operation that should be guarded",\n "parameters": [\n {\n "name": "target",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string",\n "dangerous_tool": "returns sensitive data"\n },\n "guardrails": {\n "tool": {\n "deny_tools": [\n "dangerous_tool"\n ],\n "reason": "Tool not authorized"\n }\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-guard-tool-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_guard_weather",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_guard_danger",\n "type": "function",\n "function": {\n "name": "dangerous_tool",\n "arguments": "{\\"target\\": \\"secret_db\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_guard_weather",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_guard_danger",\n "name": "dangerous_tool",\n "arguments": {\n "target": "secret_db"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_guard_weather",\n "result": "72°F sunny"\n },\n {\n "tool_call_id": "call_guard_danger",\n "result": "Tool denied by guardrail: Tool not authorized"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-guard-tool-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny. I was unable to run the dangerous operation as it was not authorized.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 72°F and sunny. I was unable to run the dangerous operation as it was not authorized.",\n "iterations": 2,\n "total_messages": 7,\n "denied_tools": [\n "dangerous_tool"\n ],\n "notes": "The tool guardrail intercepts tool calls before execution. get_weather is allowed and executes normally. dangerous_tool is denied — its tool result message contains a synthetic denial string instead of actual execution output. Both results are sent to the LLM so it can respond appropriately."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "max_iterations_exceeded",\n "description": "LLM returns tool calls on every turn for 11 iterations, exceeding MAX_ITERATIONS=10. The agent loop must raise an error.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Keep checking the weather forever."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-loop-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_01",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_01",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_01",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-loop-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_02",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_02",\n "name": "get_weather",\n "arguments": {\n "city": "London"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_02",\n "result": "59°F cloudy"\n }\n ]\n },\n {\n "turn": 3,\n "llm_response": {\n "id": "chatcmpl-loop-003",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_03",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Tokyo\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_03",\n "name": "get_weather",\n "arguments": {\n "city": "Tokyo"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_03",\n "result": "80°F humid"\n }\n ]\n },\n {\n "turn": 4,\n "llm_response": {\n "id": "chatcmpl-loop-004",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_04",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Sydney\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_04",\n "name": "get_weather",\n "arguments": {\n "city": "Sydney"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_04",\n "result": "65°F rainy"\n }\n ]\n },\n {\n "turn": 5,\n "llm_response": {\n "id": "chatcmpl-loop-005",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_05",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Berlin\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_05",\n "name": "get_weather",\n "arguments": {\n "city": "Berlin"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_05",\n "result": "55°F windy"\n }\n ]\n },\n {\n "turn": 6,\n "llm_response": {\n "id": "chatcmpl-loop-006",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_06",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Cairo\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_06",\n "name": "get_weather",\n "arguments": {\n "city": "Cairo"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_06",\n "result": "95°F dry"\n }\n ]\n },\n {\n "turn": 7,\n "llm_response": {\n "id": "chatcmpl-loop-007",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_07",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Mumbai\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_07",\n "name": "get_weather",\n "arguments": {\n "city": "Mumbai"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_07",\n "result": "88°F humid"\n }\n ]\n },\n {\n "turn": 8,\n "llm_response": {\n "id": "chatcmpl-loop-008",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_08",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Moscow\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_08",\n "name": "get_weather",\n "arguments": {\n "city": "Moscow"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_08",\n "result": "30°F snowy"\n }\n ]\n },\n {\n "turn": 9,\n "llm_response": {\n "id": "chatcmpl-loop-009",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_09",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Rio\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_09",\n "name": "get_weather",\n "arguments": {\n "city": "Rio"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_09",\n "result": "85°F tropical"\n }\n ]\n },\n {\n "turn": 10,\n "llm_response": {\n "id": "chatcmpl-loop-010",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_10",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Seoul\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_10",\n "name": "get_weather",\n "arguments": {\n "city": "Seoul"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_10",\n "result": "68°F clear"\n }\n ]\n },\n {\n "turn": 11,\n "llm_response": {\n "id": "chatcmpl-loop-011",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_loop_11",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Toronto\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_loop_11",\n "name": "get_weather",\n "arguments": {\n "city": "Toronto"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_loop_11",\n "result": "40°F foggy"\n }\n ]\n }\n ],\n "expected": {\n "error": "Agent loop exceeded 10 iterations",\n "iterations": 11\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "multi_turn_tool_calls",\n "description": "Three-turn chain: Turn 1 calls get_weather, Turn 2 calls convert_temperature with the result, Turn 3 returns the final answer.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant with access to weather and conversion tools."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris in Celsius?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city (returns Fahrenheit)",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "convert_temperature",\n "kind": "function",\n "description": "Convert a temperature between Fahrenheit and Celsius",\n "parameters": [\n {\n "name": "value",\n "kind": "float",\n "required": true\n },\n {\n "name": "from_unit",\n "kind": "string",\n "required": true\n },\n {\n "name": "to_unit",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string in Fahrenheit",\n "convert_temperature": "returns converted temperature string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-chain-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_weather_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_weather_001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_weather_001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-chain-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_convert_001",\n "type": "function",\n "function": {\n "name": "convert_temperature",\n "arguments": "{\\"value\\": 72.0, \\"from_unit\\": \\"fahrenheit\\", \\"to_unit\\": \\"celsius\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_convert_001",\n "name": "convert_temperature",\n "arguments": {\n "value": 72,\n "from_unit": "fahrenheit",\n "to_unit": "celsius"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_convert_001",\n "result": "22.2°C"\n }\n ]\n },\n {\n "turn": 3,\n "llm_response": {\n "id": "chatcmpl-chain-003",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 22.2°C and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 22.2°C and sunny.",\n "iterations": 3,\n "total_messages": 8\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "multiple_tool_calls_single_turn",\n "description": "Turn 1: LLM returns two parallel tool calls — get_weather for Paris and London. Both results appended. Turn 2: LLM summarises both.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "Compare the weather in Paris and London."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-multi-call-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_paris_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_london_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_paris_001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_london_001",\n "name": "get_weather",\n "arguments": {\n "city": "London"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_paris_001",\n "result": "72°F sunny"\n },\n {\n "tool_call_id": "call_london_001",\n "result": "59°F cloudy"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-multi-call-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Paris is 72°F and sunny, while London is 59°F and cloudy.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "Paris is 72°F and sunny, while London is 59°F and cloudy.",\n "iterations": 2,\n "total_messages": 7,\n "message_sequence": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "Compare the weather in Paris and London."\n },\n {\n "role": "assistant",\n "content": "",\n "metadata": {\n "tool_calls": [\n {\n "id": "call_paris_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_london_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"London\\"}"\n }\n }\n ]\n }\n },\n {\n "role": "tool",\n "content": "72°F sunny",\n "metadata": {\n "tool_call_id": "call_paris_001"\n }\n },\n {\n "role": "tool",\n "content": "59°F cloudy",\n "metadata": {\n "tool_call_id": "call_london_001"\n }\n },\n {\n "role": "assistant",\n "content": "Paris is 72°F and sunny, while London is 59°F and cloudy."\n }\n ]\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "no_tool_calls",\n "description": "LLM returns content immediately with no tool calls — the agent loop completes in a single iteration.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is 2 + 2?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-no-tools-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "2 + 2 equals 4.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "2 + 2 equals 4.",\n "iterations": 1,\n "total_messages": 3\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "parallel_tools_basic",\n "description": "§13.6 Parallel Tools — LLM requests 3 tool calls in one turn. All execute (potentially in parallel) and results are returned in the same order as the requests.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant with access to weather, time, and news tools."\n },\n {\n "role": "user",\n "content": "Give me the weather in Paris, the current time in Tokyo, and the latest news."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "get_time",\n "kind": "function",\n "description": "Get the current time in a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "get_news",\n "kind": "function",\n "description": "Get the latest news headlines",\n "parameters": []\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string",\n "get_time": "returns current time string",\n "get_news": "returns news headlines"\n },\n "parallel_tool_calls": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-parallel-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_par_weather",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_par_time",\n "type": "function",\n "function": {\n "name": "get_time",\n "arguments": "{\\"city\\": \\"Tokyo\\"}"\n }\n },\n {\n "id": "call_par_news",\n "type": "function",\n "function": {\n "name": "get_news",\n "arguments": "{}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_par_weather",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_par_time",\n "name": "get_time",\n "arguments": {\n "city": "Tokyo"\n }\n },\n {\n "id": "call_par_news",\n "name": "get_news",\n "arguments": {}\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_par_weather",\n "result": "72°F sunny"\n },\n {\n "tool_call_id": "call_par_time",\n "result": "3:45 PM JST"\n },\n {\n "tool_call_id": "call_par_news",\n "result": "Tech stocks rise 5%. New climate accord signed. Mars rover discovers ice."\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-parallel-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Here\'s your update: Paris is 72°F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "rust_expected_error": "parallel_tool_calls=true is not supported by the canonical Rust engine",\n "result": "Here\'s your update: Paris is 72°F and sunny. The time in Tokyo is 3:45 PM JST. Latest news: Tech stocks rise 5%, a new climate accord was signed, and the Mars rover discovered ice.",\n "iterations": 2,\n "total_messages": 8,\n "tool_execution_order": [\n "get_weather",\n "get_time",\n "get_news"\n ],\n "notes": "With parallel_tool_calls=true, all 3 tool calls may execute concurrently. Regardless of execution order, tool result messages must be appended in the same order as the original tool_calls array. The total is 8 messages: system + user + assistant(3 tool_calls) + 3 tool results + final assistant."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "parallel_tools_with_guardrail_deny",\n "description": "§13.6 Parallel Tools + §13.4 Guardrails — 3 parallel tool calls, one denied by tool guardrail. The 2 allowed tools execute, the denied tool gets a synthetic result.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Get weather, time, and run the dangerous operation."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "get_time",\n "kind": "function",\n "description": "Get the current time in a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n },\n {\n "name": "dangerous_tool",\n "kind": "function",\n "description": "A dangerous operation",\n "parameters": [\n {\n "name": "target",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string",\n "get_time": "returns current time string",\n "dangerous_tool": "returns sensitive data"\n },\n "parallel_tool_calls": true,\n "guardrails": {\n "tool": {\n "deny_tools": [\n "dangerous_tool"\n ],\n "reason": "Not authorized"\n }\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-par-guard-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_pg_weather",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "id": "call_pg_danger",\n "type": "function",\n "function": {\n "name": "dangerous_tool",\n "arguments": "{\\"target\\": \\"secret_db\\"}"\n }\n },\n {\n "id": "call_pg_time",\n "type": "function",\n "function": {\n "name": "get_time",\n "arguments": "{\\"city\\": \\"Tokyo\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_pg_weather",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n },\n {\n "id": "call_pg_danger",\n "name": "dangerous_tool",\n "arguments": {\n "target": "secret_db"\n }\n },\n {\n "id": "call_pg_time",\n "name": "get_time",\n "arguments": {\n "city": "Tokyo"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_pg_weather",\n "result": "72°F sunny"\n },\n {\n "tool_call_id": "call_pg_danger",\n "result": "Tool denied by guardrail: Not authorized"\n },\n {\n "tool_call_id": "call_pg_time",\n "result": "3:45 PM JST"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-par-guard-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Paris is 72°F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "rust_expected_error": "parallel_tool_calls=true is not supported by the canonical Rust engine",\n "result": "Paris is 72°F and sunny. The time in Tokyo is 3:45 PM JST. The dangerous operation could not be executed as it is not authorized.",\n "iterations": 2,\n "total_messages": 8,\n "denied_tools": [\n "dangerous_tool"\n ],\n "tool_execution_order": [\n "get_weather",\n "get_time"\n ],\n "notes": "The tool guardrail filters dangerous_tool before execution. get_weather and get_time execute (potentially in parallel). dangerous_tool receives a synthetic denial result. All 3 tool result messages are appended in the original tool_calls order. The denied tool is never actually called but its result slot is filled with the denial message."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "single_tool_call",\n "description": "Turn 1: LLM requests get_weather for Paris. Turn 2: after receiving the tool result, LLM returns a final content response.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-single-tool-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_abc123",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_abc123",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_abc123",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-single-tool-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The weather in Paris is 72°F and sunny.",\n "iterations": 2,\n "total_messages": 6,\n "message_sequence": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n },\n {\n "role": "assistant",\n "content": "",\n "metadata": {\n "tool_calls": [\n {\n "id": "call_abc123",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n }\n },\n {\n "role": "tool",\n "content": "72°F sunny",\n "metadata": {\n "tool_call_id": "call_abc123"\n }\n },\n {\n "role": "assistant",\n "content": "The weather in Paris is 72°F and sunny."\n }\n ]\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "steering_inject_message",\n "description": "§13.5 Steering — After turn 1 (tool call + result), a steering message is injected before turn 2\'s LLM call. The LLM sees the injected message in context.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "steering": {\n "messages": [\n {\n "inject_before_iteration": 2,\n "role": "user",\n "text": "Actually, focus on the temperature in Celsius."\n }\n ]\n },\n "on_event": true\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-steer-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_steer_w001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_steer_w001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_steer_w001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-steer-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "The temperature in Paris is approximately 22°C and it is sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "The temperature in Paris is approximately 22°C and it is sunny.",\n "iterations": 2,\n "total_messages": 7,\n "events": [\n {\n "type": "status",\n "data": {\n "message": "Starting agent loop"\n }\n },\n {\n "type": "tool_call_start",\n "data": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n },\n {\n "type": "tool_result",\n "data": {\n "name": "get_weather",\n "result": "72°F sunny"\n }\n },\n {\n "type": "messages_updated",\n "data": {\n "message_count": 5\n }\n },\n {\n "type": "status",\n "data": {\n "message": "Injecting steering message"\n }\n },\n {\n "type": "messages_updated",\n "data": {\n "message_count": 6\n }\n },\n {\n "type": "done",\n "data": {\n "response": "The temperature in Paris is approximately 22°C and it is sunny."\n }\n }\n ],\n "message_sequence": [\n {\n "role": "system",\n "content": "You are a helpful weather assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n },\n {\n "role": "assistant",\n "content": "",\n "metadata": {\n "tool_calls": [\n {\n "id": "call_steer_w001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n }\n },\n {\n "role": "tool",\n "content": "72°F sunny",\n "metadata": {\n "tool_call_id": "call_steer_w001"\n }\n },\n {\n "role": "user",\n "content": "Actually, focus on the temperature in Celsius."\n },\n {\n "role": "assistant",\n "content": "The temperature in Paris is approximately 22°C and it is sunny."\n }\n ],\n "notes": "The steering message is injected as a user message before iteration 2\'s LLM call. total_messages is 7 (system + user + assistant_tool + tool_result + steering_user + assistant_final = 6 shown, but 7 when including the final assistant). The steering message appears between the tool result and the final LLM call."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "steering_multiple_messages",\n "description": "§13.5 Steering — Two steering messages are queued and both injected before iteration 2. Tests atomic drain — both appear in order.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns weather string for the requested city"\n },\n "steering": {\n "messages": [\n {\n "inject_before_iteration": 2,\n "role": "user",\n "text": "Please respond in French."\n },\n {\n "inject_before_iteration": 2,\n "role": "user",\n "text": "Also include the humidity percentage."\n }\n ]\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-steer-multi-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_steer_m001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_steer_m001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_steer_m001",\n "result": "72°F sunny, 45% humidity"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-steer-multi-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "Le temps à Paris est de 22°C et ensoleillé avec une humidité de 45%.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "Le temps à Paris est de 22°C et ensoleillé avec une humidité de 45%.",\n "iterations": 2,\n "total_messages": 8,\n "message_sequence": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n },\n {\n "role": "assistant",\n "content": "",\n "metadata": {\n "tool_calls": [\n {\n "id": "call_steer_m001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n }\n },\n {\n "role": "tool",\n "content": "72°F sunny, 45% humidity",\n "metadata": {\n "tool_call_id": "call_steer_m001"\n }\n },\n {\n "role": "user",\n "content": "Please respond in French."\n },\n {\n "role": "user",\n "content": "Also include the humidity percentage."\n },\n {\n "role": "assistant",\n "content": "Le temps à Paris est de 22°C et ensoleillé avec une humidité de 45%."\n }\n ],\n "notes": "Both steering messages are drained atomically and inserted in order before iteration 2\'s LLM call. They appear as consecutive user messages between the tool result and the final assistant response. total_messages is 8 (system + user + assistant_tool + tool_result + steer1 + steer2 + assistant_final = 7 content messages, but counting from 1-indexed gives 8 including all roles)."\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "tool_not_registered_error",\n "description": "LLM calls a tool named \'unknown_tool\' that has no registered handler in tool_functions. The agent loop must raise a ValueError.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "Do something with an unknown tool."\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-unknown-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_unknown_001",\n "type": "function",\n "function": {\n "name": "unknown_tool",\n "arguments": "{\\"query\\": \\"test\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_unknown_001",\n "name": "unknown_tool",\n "arguments": {\n "query": "test"\n }\n }\n ]\n }\n ],\n "expected": {\n "error": "Tool not registered: unknown_tool",\n "error_type": "ValueError"\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "run",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "tool_result_message_format",\n "description": "After executing a tool, the tool result message appended to the conversation must have role=\'tool\', content as a TextPart with the stringified result, and metadata.tool_call_id matching the original call id.",\n "stage": "agent",\n "input": {\n "messages": [\n {\n "role": "system",\n "content": "You are a helpful assistant."\n },\n {\n "role": "user",\n "content": "What is the weather?"\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get the current weather for a city",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "tool_functions": {\n "get_weather": "returns a weather string"\n }\n },\n "sequence": [\n {\n "turn": 1,\n "llm_response": {\n "id": "chatcmpl-format-001",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": null,\n "tool_calls": [\n {\n "id": "call_fmt_001",\n "type": "function",\n "function": {\n "name": "get_weather",\n "arguments": "{\\"city\\": \\"Paris\\"}"\n }\n }\n ]\n },\n "finish_reason": "tool_calls"\n }\n ]\n },\n "expected_tool_calls": [\n {\n "id": "call_fmt_001",\n "name": "get_weather",\n "arguments": {\n "city": "Paris"\n }\n }\n ],\n "tool_results": [\n {\n "tool_call_id": "call_fmt_001",\n "result": "72°F sunny"\n }\n ]\n },\n {\n "turn": 2,\n "llm_response": {\n "id": "chatcmpl-format-002",\n "object": "chat.completion",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "It is 72°F and sunny.",\n "tool_calls": null\n },\n "finish_reason": "stop"\n }\n ]\n },\n "expected_tool_calls": null\n }\n ],\n "expected": {\n "result": "It is 72°F and sunny.",\n "iterations": 2,\n "tool_result_message": {\n "role": "tool",\n "content": [\n {\n "type": "text",\n "text": "72°F sunny"\n }\n ],\n "metadata": {\n "tool_call_id": "call_fmt_001"\n }\n }\n },\n "operation": "run"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "runTurn",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "cancel_before_context",\n "stage": "turn",\n "input": {\n "messages": [\n {\n "role": "user",\n "content": "Do not invoke the model"\n }\n ],\n "model": [],\n "cancelBeforeRun": true\n },\n "expected": {\n "status": "cancelled",\n "iterations": 0,\n "snapshots": 0,\n "toolResults": 0,\n "eventKinds": [\n "turn_started",\n "turn_cancelled"\n ]\n },\n "operation": "runTurn"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "runTurn",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "delegated_provider_state",\n "stage": "turn",\n "input": {\n "messages": [\n {\n "role": "user",\n "content": "Continue a provider-managed response"\n }\n ],\n "model": [\n {\n "tools": [\n {\n "id": "call-state",\n "name": "echo",\n "arguments": {\n "value": "state"\n }\n }\n ],\n "nextPortability": "delegated",\n "delegatedState": [\n {\n "provider": "openai",\n "kind": "previous_response",\n "id": "resp_123"\n }\n ]\n },\n {\n "output": "continued"\n }\n ],\n "toolOutputs": {\n "call-state": "state"\n }\n },\n "expected": {\n "status": "success",\n "output": "continued",\n "iterations": 2,\n "snapshots": 2,\n "snapshotStablePrefixes": [\n 1,\n 1\n ],\n "snapshotPortability": [\n "portable",\n "delegated"\n ],\n "toolResults": 1,\n "toolResultOrder": [\n "call-state"\n ],\n "commitPortability": "delegated",\n "delegatedState": 1\n },\n "operation": "runTurn"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "runTurn",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "final_output",\n "stage": "turn",\n "input": {\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ],\n "model": [\n {\n "output": "Hello back"\n }\n ]\n },\n "expected": {\n "status": "success",\n "output": "Hello back",\n "iterations": 1,\n "snapshots": 1,\n "snapshotStablePrefixes": [\n 1\n ],\n "toolResults": 0,\n "eventKinds": [\n "turn_started",\n "context_prepared",\n "model_invocation_started",\n "model_invocation_completed",\n "checkpoint_created",\n "turn_committed",\n "post_commit_started",\n "post_commit_completed"\n ]\n },\n "operation": "runTurn"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "runTurn",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "ordered_tool_round",\n "stage": "turn",\n "input": {\n "messages": [\n {\n "role": "user",\n "content": "Add two values"\n }\n ],\n "model": [\n {\n "assistant": "I will use the tools.",\n "tools": [\n {\n "id": "call-a",\n "name": "echo",\n "arguments": {\n "value": "A"\n }\n },\n {\n "id": "call-b",\n "name": "echo",\n "arguments": {\n "value": "B"\n }\n }\n ]\n },\n {\n "output": "A then B"\n }\n ],\n "toolOutputs": {\n "call-a": "A",\n "call-b": "B"\n }\n },\n "expected": {\n "status": "success",\n "output": "A then B",\n "iterations": 2,\n "snapshots": 2,\n "snapshotStablePrefixes": [\n 1,\n 1\n ],\n "toolResults": 2,\n "toolResultOrder": [\n "call-a",\n "call-b"\n ],\n "eventKinds": [\n "turn_started",\n "context_prepared",\n "model_invocation_started",\n "model_invocation_completed",\n "checkpoint_created",\n "permission_requested",\n "permission_resolved",\n "tool_execution_started",\n "tool_execution_completed",\n "checkpoint_created",\n "permission_requested",\n "permission_resolved",\n "tool_execution_started",\n "tool_execution_completed",\n "checkpoint_created",\n "tool_result_committed",\n "tool_result_committed",\n "conversation_updated",\n "checkpoint_created",\n "context_prepared",\n "model_invocation_started",\n "model_invocation_completed",\n "checkpoint_created",\n "turn_committed",\n "post_commit_started",\n "post_commit_completed"\n ]\n },\n "operation": "runTurn"\n }\n },\n {\n "contract": "TurnConformance",\n "operation": "runTurn",\n "params": {\n "input": "unknown"\n },\n "returns": "unknown",\n "vector": {\n "name": "permission_denial_is_model_visible",\n "stage": "turn",\n "input": {\n "messages": [\n {\n "role": "user",\n "content": "Read the protected resource"\n }\n ],\n "model": [\n {\n "tools": [\n {\n "id": "call-denied",\n "name": "protected",\n "arguments": {}\n }\n ]\n },\n {\n "output": "Permission was denied"\n }\n ],\n "denyTools": [\n "protected"\n ]\n },\n "expected": {\n "status": "success",\n "output": "Permission was denied",\n "iterations": 2,\n "snapshots": 2,\n "snapshotStablePrefixes": [\n 1,\n 1\n ],\n "toolResults": 1,\n "toolResultOrder": [\n "call-denied"\n ]\n },\n "operation": "runTurn"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_image_format",\n "description": "§7.5 — Anthropic: ImagePart uses base64 source block with media_type, not image_url format.",\n "stage": "wire",\n "provider": "anthropic",\n "targetApi": "chat",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "model_id": "claude-3",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Describe this image"\n },\n {\n "kind": "image",\n "value": "base64data",\n "mediaType": "image/png"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "maxOutputTokens": 4096\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "claude-3",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Describe this image"\n },\n {\n "type": "image",\n "source": {\n "type": "base64",\n "media_type": "image/png",\n "data": "base64data"\n }\n }\n ]\n }\n ],\n "max_tokens": 4096\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_max_tokens_required",\n "description": "§7.5 — Anthropic: when maxOutputTokens is not set, max_tokens MUST default to 4096 (required by API).",\n "stage": "wire",\n "provider": "anthropic",\n "targetApi": "chat",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "model_id": "claude-3",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "claude-3",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Hello"\n }\n ]\n }\n ],\n "max_tokens": 4096\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_options",\n "description": "Anthropic: option names map correctly (topK, stopSequences)",\n "stage": "wire",\n "provider": "anthropic",\n "targetApi": "chat",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "model_id": "claude-sonnet-4-20250514",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hi"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "temperature": 0.5,\n "topP": 0.9,\n "topK": 40,\n "maxOutputTokens": 2000,\n "stopSequences": [\n "END"\n ]\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "claude-sonnet-4-20250514",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Hi"\n }\n ]\n }\n ],\n "temperature": 0.5,\n "top_p": 0.9,\n "top_k": 40,\n "max_tokens": 2000,\n "stop_sequences": [\n "END"\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_system_separate",\n "description": "§7.5 — Anthropic: system message extracted to top-level \'system\' field; messages array contains only non-system messages.",\n "stage": "wire",\n "provider": "anthropic",\n "targetApi": "chat",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "model_id": "claude-3",\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Be helpful"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "maxOutputTokens": 4096\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "claude-3",\n "system": "Be helpful",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Hello"\n }\n ]\n }\n ],\n "max_tokens": 4096\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "anthropic_tool_wire",\n "description": "Anthropic: tools use input_schema (not parameters), name at top level",\n "stage": "wire",\n "provider": "anthropic",\n "targetApi": "chat",\n "input": {\n "provider": "anthropic",\n "apiType": "chat",\n "model_id": "claude-sonnet-4-20250514",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Weather?"\n }\n ]\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get weather",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "claude-sonnet-4-20250514",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Weather?"\n }\n ]\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "description": "Get weather",\n "input_schema": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string"\n }\n },\n "required": [\n "city"\n ]\n }\n }\n ],\n "max_tokens": 4096\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_audio_mp3",\n "description": "§7.1.2 — AudioPart with mediaType audio/mpeg MUST map to format \'mp3\', not \'mpeg\'.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "audio",\n "value": "base64data",\n "mediaType": "audio/mpeg"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "input_audio",\n "input_audio": {\n "data": "base64data",\n "format": "mp3"\n }\n }\n ]\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_audio_part",\n "description": "§7.1.2 — AudioPart with mediaType audio/wav maps to input_audio with format \'wav\'.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "audio",\n "value": "base64data",\n "mediaType": "audio/wav"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "input_audio",\n "input_audio": {\n "data": "base64data",\n "format": "wav"\n }\n }\n ]\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_image_base64",\n "description": "Base64 data URI image part preserved as-is in image_url",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Describe this"\n },\n {\n "kind": "image",\n "value": "data:image/png;base64,iVBOR..."\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Describe this"\n },\n {\n "type": "image_url",\n "image_url": {\n "url": "data:image/png;base64,iVBOR..."\n }\n }\n ]\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_image_part",\n "description": "Image content part converted to OpenAI image_url format",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "What\'s in this image?"\n },\n {\n "kind": "image",\n "value": "https://example.com/photo.jpg"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "What\'s in this image?"\n },\n {\n "type": "image_url",\n "image_url": {\n "url": "https://example.com/photo.jpg"\n }\n }\n ]\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_multipart_content",\n "description": "§7.1.1/§7.1.2 — Message with TextPart + ImagePart MUST produce an array of typed content blocks.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Describe this image"\n },\n {\n "kind": "image",\n "value": "https://img.png"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "type": "text",\n "text": "Describe this image"\n },\n {\n "type": "image_url",\n "image_url": {\n "url": "https://img.png"\n }\n }\n ]\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_simple",\n "description": "§7.1.7 — Two messages (system + user), no tools, no options. Minimal chat request.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Be helpful"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "system",\n "content": "Be helpful"\n },\n {\n "role": "user",\n "content": "Hello"\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_single_text_optimized",\n "description": "§7.1.1 — Message with exactly 1 TextPart MUST produce a plain string content, not an array.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "What is 2+2?"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "What is 2+2?"\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "chat_with_options",\n "description": "§7.1.5 — All standard ModelOptions mapped to OpenAI parameter names.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "temperature": 0.5,\n "maxOutputTokens": 100,\n "topP": 0.9,\n "frequencyPenalty": 0.1,\n "presencePenalty": 0.2,\n "seed": 42,\n "stopSequences": [\n "END"\n ]\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ],\n "temperature": 0.5,\n "max_completion_tokens": 100,\n "top_p": 0.9,\n "frequency_penalty": 0.1,\n "presence_penalty": 0.2,\n "seed": 42,\n "stop": [\n "END"\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "embedding_wire",\n "description": "§7.2 — Embedding request: text extracted from messages, single string input (not array).",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "embedding",\n "input": {\n "provider": "openai",\n "apiType": "embedding",\n "model_id": "text-embedding-3-small",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello world"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "text-embedding-3-small",\n "input": "Hello world"\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "image_wire",\n "description": "§7.3 — Image generation request: prompt extracted from last user message.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "image",\n "input": {\n "provider": "openai",\n "apiType": "image",\n "model_id": "dall-e-3",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "A cat"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "dall-e-3",\n "prompt": "A cat"\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "kind_to_json_type_mapping",\n "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Test"\n }\n ]\n }\n ],\n "tools": [\n {\n "kind": "function",\n "name": "test_types",\n "description": "Tests all type mappings",\n "parameters": [\n {\n "name": "a_string",\n "kind": "string"\n },\n {\n "name": "an_integer",\n "kind": "integer"\n },\n {\n "name": "a_float",\n "kind": "float"\n },\n {\n "name": "a_boolean",\n "kind": "boolean"\n },\n {\n "name": "an_array",\n "kind": "array",\n "items": {\n "kind": "string"\n }\n },\n {\n "name": "an_object",\n "kind": "object"\n }\n ]\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Test"\n }\n ],\n "tools": [\n {\n "type": "function",\n "function": {\n "name": "test_types",\n "description": "Tests all type mappings",\n "parameters": {\n "type": "object",\n "properties": {\n "a_string": {\n "type": "string"\n },\n "an_integer": {\n "type": "integer"\n },\n "a_float": {\n "type": "number"\n },\n "a_boolean": {\n "type": "boolean"\n },\n "an_array": {\n "type": "array",\n "items": {\n "type": "string"\n }\n },\n "an_object": {\n "type": "object"\n }\n }\n }\n }\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "options_additional_properties",\n "description": "§7.1.5 — additionalProperties from ModelOptions MUST be merged into the request as top-level keys.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "additionalProperties": {\n "logprobs": true\n }\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ],\n "logprobs": true\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "options_max_completion_tokens",\n "description": "§7.1.5 — maxOutputTokens MUST map to max_completion_tokens, NOT the deprecated max_tokens.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "maxOutputTokens": 500\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ],\n "max_completion_tokens": 500\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "options_stop_sequences",\n "description": "§7.1.5 — stopSequences MUST map to \'stop\' in the OpenAI wire format.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {\n "stopSequences": [\n ".",\n "!"\n ]\n },\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ],\n "stop": [\n ".",\n "!"\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_simple",\n "description": "Responses API: system becomes instructions, messages become input items",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "responses",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "model_id": "gpt-4o",\n "messages": [\n {\n "role": "system",\n "content": [\n {\n "kind": "text",\n "value": "Be helpful"\n }\n ]\n },\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o",\n "instructions": "Be helpful",\n "input": [\n {\n "role": "user",\n "content": "Hello"\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_structured_output",\n "description": "Responses API: structured output uses text.format.json_schema",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "responses",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "model_id": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Give me data"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": [\n {\n "name": "city",\n "kind": "string"\n },\n {\n "name": "temp",\n "kind": "integer"\n }\n ]\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o",\n "input": [\n {\n "role": "user",\n "content": "Give me data"\n }\n ],\n "text": {\n "format": {\n "type": "json_schema",\n "name": "structured_output",\n "schema": {\n "type": "object",\n "properties": {\n "city": {\n "type": [\n "string",\n "null"\n ]\n },\n "temp": {\n "type": [\n "integer",\n "null"\n ]\n }\n },\n "required": [\n "city",\n "temp"\n ],\n "additionalProperties": false\n },\n "strict": true\n }\n }\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "responses_with_tools",\n "description": "Responses API: tools use flat format (no nested function key)",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "responses",\n "input": {\n "provider": "openai",\n "apiType": "responses",\n "model_id": "gpt-4o",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Weather?"\n }\n ]\n }\n ],\n "tools": [\n {\n "name": "get_weather",\n "kind": "function",\n "description": "Get weather",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o",\n "input": [\n {\n "role": "user",\n "content": "Weather?"\n }\n ],\n "tools": [\n {\n "type": "function",\n "name": "get_weather",\n "description": "Get weather",\n "parameters": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string"\n }\n },\n "required": [\n "city"\n ]\n }\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "structured_output",\n "description": "§7.1.6 — outputs converted to response_format with json_schema, strict=true, additionalProperties=false.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "What is the weather in Paris?"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n },\n {\n "name": "temp",\n "kind": "integer",\n "required": true\n }\n ]\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "What is the weather in Paris?"\n }\n ],\n "response_format": {\n "type": "json_schema",\n "json_schema": {\n "name": "structured_output",\n "strict": true,\n "schema": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string"\n },\n "temp": {\n "type": "integer"\n }\n },\n "required": [\n "city",\n "temp"\n ],\n "additionalProperties": false\n }\n }\n }\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "structured_output_nested_optional",\n "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4o-mini",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Choose a visual style."\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": [\n {\n "name": "style",\n "kind": "object",\n "required": true,\n "properties": [\n {\n "name": "color",\n "kind": "string",\n "required": true\n },\n {\n "name": "border",\n "kind": "string",\n "required": false\n }\n ]\n }\n ]\n },\n "expected": {\n "request_body": {\n "model": "gpt-4o-mini",\n "messages": [\n {\n "role": "user",\n "content": "Choose a visual style."\n }\n ],\n "response_format": {\n "type": "json_schema",\n "json_schema": {\n "name": "structured_output",\n "strict": true,\n "schema": {\n "type": "object",\n "properties": {\n "style": {\n "type": "object",\n "properties": {\n "color": {\n "type": "string"\n },\n "border": {\n "type": [\n "string",\n "null"\n ]\n }\n },\n "required": [\n "color",\n "border"\n ],\n "additionalProperties": false\n }\n },\n "required": [\n "style"\n ],\n "additionalProperties": false\n }\n }\n }\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "tools_bindings_stripped",\n "description": "§7.1.3 — Parameters listed in bindings MUST be stripped from wire tools (properties AND required).",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Get the weather"\n }\n ]\n }\n ],\n "tools": [\n {\n "kind": "function",\n "name": "get_weather",\n "description": "Get weather",\n "parameters": [\n {\n "name": "location",\n "kind": "string",\n "required": true\n },\n {\n "name": "unit",\n "kind": "string",\n "required": true\n }\n ],\n "bindings": {\n "unit": {\n "input": "preferred_unit"\n }\n }\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Get the weather"\n }\n ],\n "tools": [\n {\n "type": "function",\n "function": {\n "name": "get_weather",\n "description": "Get weather",\n "parameters": {\n "type": "object",\n "properties": {\n "location": {\n "type": "string"\n }\n },\n "required": [\n "location"\n ]\n }\n }\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "tools_function_wire",\n "description": "§7.1.3 — FunctionTool projected as OpenAI function definition with JSON Schema parameters.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "What is the weather?"\n }\n ]\n }\n ],\n "tools": [\n {\n "kind": "function",\n "name": "get_weather",\n "description": "Get weather",\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "What is the weather?"\n }\n ],\n "tools": [\n {\n "type": "function",\n "function": {\n "name": "get_weather",\n "description": "Get weather",\n "parameters": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string"\n }\n },\n "required": [\n "city"\n ]\n }\n }\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "tools_null_when_empty",\n "description": "§7.1.3 — When no tools are defined, the tools key MUST be absent from the request entirely.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Hello"\n }\n ]\n }\n ],\n "tools": [],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Hello"\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n },\n {\n "contract": "WireConformance",\n "operation": "toRequest",\n "params": {\n "agent": "Agent",\n "messages": "Message[]"\n },\n "returns": "unknown",\n "vector": {\n "name": "tools_strict_mode",\n "description": "§7.1.3 — FunctionTool with strict=true: strict lives on function def, additionalProperties=false in parameters schema.",\n "stage": "wire",\n "provider": "openai",\n "targetApi": "chat",\n "input": {\n "provider": "openai",\n "apiType": "chat",\n "model_id": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": [\n {\n "kind": "text",\n "value": "Get the weather"\n }\n ]\n }\n ],\n "tools": [\n {\n "kind": "function",\n "name": "get_weather",\n "description": "Get weather",\n "strict": true,\n "parameters": [\n {\n "name": "city",\n "kind": "string",\n "required": true\n }\n ]\n }\n ],\n "options": {},\n "outputs": []\n },\n "expected": {\n "request_body": {\n "model": "gpt-4",\n "messages": [\n {\n "role": "user",\n "content": "Get the weather"\n }\n ],\n "tools": [\n {\n "type": "function",\n "function": {\n "name": "get_weather",\n "description": "Get weather",\n "strict": true,\n "parameters": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string"\n }\n },\n "required": [\n "city"\n ],\n "additionalProperties": false\n }\n }\n }\n ]\n }\n },\n "operation": "toRequest"\n }\n }\n]' ) diff --git a/runtime/rust/prompty-anthropic/README.md b/runtime/rust/prompty-anthropic/README.md index 76441e346..4759875b4 100644 --- a/runtime/rust/prompty-anthropic/README.md +++ b/runtime/rust/prompty-anthropic/README.md @@ -28,6 +28,9 @@ model: apiKey: ${env:ANTHROPIC_API_KEY} ``` +`ANTHROPIC_BASE_URL` optionally overrides the default `https://api.anthropic.com` +base URL. A non-empty `model.connection.endpoint` takes precedence. + ## License MIT — see [LICENSE](LICENSE) for details. diff --git a/runtime/rust/prompty-anthropic/src/endpoint.rs b/runtime/rust/prompty-anthropic/src/endpoint.rs new file mode 100644 index 000000000..a74d36d1e --- /dev/null +++ b/runtime/rust/prompty-anthropic/src/endpoint.rs @@ -0,0 +1,93 @@ +//! Resolve Anthropic API endpoints consistently across provider operations. + +use serde_json::Value; + +const DEFAULT_BASE_URL: &str = "https://api.anthropic.com"; + +pub(crate) fn build_api_url(connection: &Value, resource: &str) -> String { + let endpoint = connection + .get("endpoint") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) + .or_else(|| { + std::env::var("ANTHROPIC_BASE_URL") + .ok() + .filter(|value| !value.is_empty()) + }) + .unwrap_or_else(|| DEFAULT_BASE_URL.to_string()); + + let base = endpoint.trim_end_matches('/'); + if base.ends_with("/v1") { + format!("{base}/{resource}") + } else { + format!("{base}/v1/{resource}") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use serial_test::serial; + + struct BaseUrlGuard(Option); + + impl BaseUrlGuard { + fn set(value: Option<&str>) -> Self { + let original = std::env::var_os("ANTHROPIC_BASE_URL"); + match value { + Some(value) => unsafe { std::env::set_var("ANTHROPIC_BASE_URL", value) }, + None => unsafe { std::env::remove_var("ANTHROPIC_BASE_URL") }, + } + Self(original) + } + } + + impl Drop for BaseUrlGuard { + fn drop(&mut self) { + match self.0.take() { + Some(value) => unsafe { std::env::set_var("ANTHROPIC_BASE_URL", value) }, + None => unsafe { std::env::remove_var("ANTHROPIC_BASE_URL") }, + } + } + } + + #[test] + #[serial] + fn default_endpoint_is_used_when_unconfigured() { + let _guard = BaseUrlGuard::set(None); + + assert_eq!( + build_api_url(&json!({}), "messages"), + "https://api.anthropic.com/v1/messages" + ); + } + + #[test] + #[serial] + fn connection_endpoint_takes_precedence_over_environment() { + let _guard = BaseUrlGuard::set(Some("https://environment.example/v1")); + let connection = json!({"endpoint": "https://connection.example/v1/"}); + + assert_eq!( + build_api_url(&connection, "messages"), + "https://connection.example/v1/messages" + ); + } + + #[test] + #[serial] + fn environment_endpoint_is_used_without_duplicate_v1() { + let _guard = BaseUrlGuard::set(Some("https://environment.example/v1/")); + + assert_eq!( + build_api_url(&json!({}), "messages"), + "https://environment.example/v1/messages" + ); + assert_eq!( + build_api_url(&json!({}), "models"), + "https://environment.example/v1/models" + ); + } +} diff --git a/runtime/rust/prompty-anthropic/src/executor.rs b/runtime/rust/prompty-anthropic/src/executor.rs index 927e3a105..a8613cca5 100644 --- a/runtime/rust/prompty-anthropic/src/executor.rs +++ b/runtime/rust/prompty-anthropic/src/executor.rs @@ -198,13 +198,7 @@ fn resolve_connection( fn build_url(agent: &Agent) -> Result { let conn = resolve_connection(agent)?; - let endpoint = conn - .get("endpoint") - .and_then(|e| e.as_str()) - .unwrap_or("https://api.anthropic.com"); - - let base = endpoint.trim_end_matches('/'); - Ok(format!("{base}/v1/messages")) + Ok(crate::endpoint::build_api_url(&conn, "messages")) } fn get_api_key(agent: &Agent) -> Result { @@ -375,8 +369,14 @@ impl Stream for AnthropicSseParser { return Poll::Ready(None); } Poll::Ready(None) => { + self.pending.push_back(serde_json::json!({ + "error": { + "type": "sse_transport_error", + "message": "Anthropic SSE stream ended before message_stop", + } + })); self.done = true; - return Poll::Ready(None); + return Poll::Ready(self.pending.pop_front()); } Poll::Pending => { return Poll::Pending; @@ -394,6 +394,24 @@ mod tests { use serde_json::json; use serial_test::serial; + struct RemovedBaseUrl(Option); + + impl RemovedBaseUrl { + fn new() -> Self { + let previous = std::env::var_os("ANTHROPIC_BASE_URL"); + unsafe { std::env::remove_var("ANTHROPIC_BASE_URL") }; + Self(previous) + } + } + + impl Drop for RemovedBaseUrl { + fn drop(&mut self) { + if let Some(value) = self.0.take() { + unsafe { std::env::set_var("ANTHROPIC_BASE_URL", value) }; + } + } + } + fn make_agent(model_json: Value) -> Agent { let mut data = json!({ "name": "test", @@ -407,6 +425,7 @@ mod tests { #[test] #[serial] fn test_build_url_default() { + let _base_url = RemovedBaseUrl::new(); let agent = make_agent(json!({"id": "claude-3", "provider": "anthropic"})); let url = build_url(&agent).unwrap(); assert_eq!(url, "https://api.anthropic.com/v1/messages"); @@ -428,6 +447,43 @@ mod tests { assert_eq!(url, "https://custom.anthropic.com/v1/messages"); } + #[test] + #[serial] + fn test_build_url_custom_endpoint_with_v1() { + let agent = make_agent(json!({ + "id": "claude-3", + "provider": "anthropic", + "connection": { + "kind": "key", + "endpoint": "https://custom.anthropic.com/v1/", + "apiKey": "test-key" + } + })); + let url = build_url(&agent).unwrap(); + assert_eq!(url, "https://custom.anthropic.com/v1/messages"); + } + + #[tokio::test] + async fn test_sse_parser_reports_premature_eof() { + use futures::StreamExt; + + let inner = futures::stream::iter(vec![Ok::(Bytes::from( + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}\n\n", + ))]); + let mut stream = AnthropicSseParser::new(inner); + + assert_eq!(stream.next().await.unwrap()["type"], "content_block_delta"); + let failure = stream.next().await.unwrap(); + assert_eq!(failure["error"]["type"], "sse_transport_error"); + assert!( + failure["error"]["message"] + .as_str() + .unwrap() + .contains("before message_stop") + ); + assert!(stream.next().await.is_none()); + } + #[test] #[serial] fn test_get_api_key_from_connection() { diff --git a/runtime/rust/prompty-anthropic/src/lib.rs b/runtime/rust/prompty-anthropic/src/lib.rs index c7d1d0705..c9258c370 100644 --- a/runtime/rust/prompty-anthropic/src/lib.rs +++ b/runtime/rust/prompty-anthropic/src/lib.rs @@ -18,6 +18,8 @@ //! // Now invoke/turn will use Anthropic for agents with provider="anthropic" //! ``` +mod endpoint; + pub mod executor; pub mod models; pub mod processor; diff --git a/runtime/rust/prompty-anthropic/src/models.rs b/runtime/rust/prompty-anthropic/src/models.rs index e6831211a..ca36121c5 100644 --- a/runtime/rust/prompty-anthropic/src/models.rs +++ b/runtime/rust/prompty-anthropic/src/models.rs @@ -21,13 +21,7 @@ const ANTHROPIC_VERSION: &str = "2023-06-01"; /// Build the models endpoint URL from a connection JSON value. fn build_models_url(connection: &Value) -> String { - let endpoint = connection - .get("endpoint") - .and_then(|e| e.as_str()) - .unwrap_or("https://api.anthropic.com"); - - let base = endpoint.trim_end_matches('/'); - format!("{base}/v1/models") + crate::endpoint::build_api_url(connection, "models") } /// Extract the API key from the connection or fall back to `ANTHROPIC_API_KEY`. @@ -280,7 +274,9 @@ mod tests { } #[test] + #[serial] fn test_build_models_url_default() { + let _env = RemovedEnv::new("ANTHROPIC_BASE_URL"); let conn = serde_json::json!({}); let url = build_models_url(&conn); assert_eq!(url, "https://api.anthropic.com/v1/models"); diff --git a/runtime/rust/prompty-anthropic/src/processor.rs b/runtime/rust/prompty-anthropic/src/processor.rs index a4f11a972..36a2eede6 100644 --- a/runtime/rust/prompty-anthropic/src/processor.rs +++ b/runtime/rust/prompty-anthropic/src/processor.rs @@ -216,7 +216,7 @@ use std::collections::BTreeMap; use std::pin::Pin; use std::task::{Context, Poll}; -use prompty::types::{StreamChunk, Usage}; +use prompty::types::{StreamChunk, StreamFailure, Usage}; /// Anthropic stream processor — converts SSE JSON events into `StreamChunk` items. /// @@ -270,6 +270,27 @@ impl futures::Stream for AnthropicStreamProcessor { AnthropicStreamPhase::Streaming => { match this.inner.as_mut().poll_next(cx) { Poll::Ready(Some(event)) => { + if let Some(error) = event.get("error").and_then(Value::as_object) { + this.phase = AnthropicStreamPhase::Done; + let error_type = + error.get("type").and_then(Value::as_str).unwrap_or(""); + let provider_message = error + .get("message") + .and_then(Value::as_str) + .unwrap_or("Anthropic stream failed"); + let message = if error_type.is_empty() { + provider_message.to_string() + } else { + format!("Anthropic stream error ({error_type}): {provider_message}") + }; + let failure = if error_type.starts_with("sse_") { + StreamFailure::Indeterminate(message) + } else { + StreamFailure::Determinate(message) + }; + return Poll::Ready(Some(StreamChunk::Failure(failure))); + } + let event_type = event.get("type").and_then(Value::as_str).unwrap_or(""); match event_type { @@ -493,6 +514,68 @@ mod tests { ); } + #[tokio::test] + async fn test_stream_provider_error_is_determinate() { + use futures::StreamExt; + + let chunks = vec![ + json!({ + "type": "content_block_delta", + "delta": {"type": "text_delta", "text": "partial"} + }), + json!({ + "type": "error", + "error": { + "type": "overloaded_error", + "message": "Overloaded" + } + }), + json!({"type": "message_delta", "usage": {"output_tokens": 5}}), + ]; + let mut stream = AnthropicStreamProcessor::new(futures::stream::iter(chunks)); + + assert!(matches!( + stream.next().await, + Some(StreamChunk::Text(value)) if value == "partial" + )); + assert!(matches!( + stream.next().await, + Some(StreamChunk::Failure(StreamFailure::Determinate(message))) + if message.contains("overloaded_error") && message.contains("Overloaded") + )); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + async fn test_stream_transport_error_is_indeterminate() { + use futures::StreamExt; + + let chunks = vec![ + json!({ + "type": "content_block_delta", + "delta": {"type": "text_delta", "text": "partial"} + }), + json!({ + "error": { + "type": "sse_transport_error", + "message": "connection reset" + } + }), + ]; + let mut stream = AnthropicStreamProcessor::new(futures::stream::iter(chunks)); + + assert!(matches!( + stream.next().await, + Some(StreamChunk::Text(value)) if value == "partial" + )); + assert!(matches!( + stream.next().await, + Some(StreamChunk::Failure(StreamFailure::Indeterminate(message))) + if message.contains("connection reset") + )); + assert!(stream.next().await.is_none()); + } + #[tokio::test] async fn test_process_text_response() { let agent = make_agent(); diff --git a/runtime/rust/prompty-openai/src/wire.rs b/runtime/rust/prompty-openai/src/wire.rs index c73bd1cbd..a3fe0c998 100644 --- a/runtime/rust/prompty-openai/src/wire.rs +++ b/runtime/rust/prompty-openai/src/wire.rs @@ -413,7 +413,7 @@ fn property_to_json_schema(prop: &Property, strict: bool) -> Result panic!("Expected object or JSON string, got: {other:?}"), }; - assert!(obj.contains_key("city"), "missing 'city' field: {obj:?}"); + let style = obj + .get("style") + .and_then(Value::as_object) + .expect("missing object 'style' field"); + assert!(style.get("color").is_some_and(Value::is_string)); assert!( - obj.contains_key("country"), - "missing 'country' field: {obj:?}" + style.contains_key("border"), + "missing 'border' field: {style:?}" ); assert!( - obj.contains_key("population"), - "missing 'population' field: {obj:?}" + style["border"].is_null() || style["border"].is_string(), + "'border' must be nullable: {style:?}" ); eprintln!("Structured output: {obj:?}"); } diff --git a/runtime/rust/prompty-openai/tests/wire_vectors.rs b/runtime/rust/prompty-openai/tests/wire_vectors.rs index 2ade16842..4a5562216 100644 --- a/runtime/rust/prompty-openai/tests/wire_vectors.rs +++ b/runtime/rust/prompty-openai/tests/wire_vectors.rs @@ -278,6 +278,7 @@ wire_test!(image_wire); wire_test!(kind_to_json_type_mapping); wire_test!(chat_image_part); wire_test!(structured_output); +wire_test!(structured_output_nested_optional); wire_test!(options_max_completion_tokens); wire_test!(options_stop_sequences); wire_test!(options_additional_properties); @@ -381,7 +382,7 @@ fn set_row_visual_schema_supports_nullable_unions_and_nested_optionality() { assert_eq!(schema["properties"]["row"]["type"], "object"); assert_eq!( schema["properties"]["row"]["required"], - json!(["color", "fill"]) + json!(["color", "border", "fill"]) ); assert_eq!( schema["properties"]["row"]["properties"]["color"]["type"], @@ -409,7 +410,7 @@ fn set_row_visual_schema_supports_nullable_unions_and_nested_optionality() { ); assert_eq!( schema["properties"]["row"]["properties"]["fill"]["anyOf"][1]["required"], - json!(["theme"]) + json!(["theme", "tint"]) ); assert_no_empty_type(&schema); } diff --git a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts index a254ce95d..a38150033 100644 --- a/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts +++ b/runtime/typescript/packages/core/tests/model/vector-conformance.test.ts @@ -9637,6 +9637,99 @@ const vectors = [ operation: "toRequest", }, }, + { + contract: "WireConformance", + operation: "toRequest", + params: { + agent: "Agent", + messages: "Message[]", + }, + returns: "unknown", + vector: { + name: "structured_output_nested_optional", + description: + "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", + stage: "wire", + provider: "openai", + targetApi: "chat", + input: { + provider: "openai", + apiType: "chat", + model_id: "gpt-4o-mini", + messages: [ + { + role: "user", + content: [ + { + kind: "text", + value: "Choose a visual style.", + }, + ], + }, + ], + tools: [], + options: {}, + outputs: [ + { + name: "style", + kind: "object", + required: true, + properties: [ + { + name: "color", + kind: "string", + required: true, + }, + { + name: "border", + kind: "string", + required: false, + }, + ], + }, + ], + }, + expected: { + request_body: { + model: "gpt-4o-mini", + messages: [ + { + role: "user", + content: "Choose a visual style.", + }, + ], + response_format: { + type: "json_schema", + json_schema: { + name: "structured_output", + strict: true, + schema: { + type: "object", + properties: { + style: { + type: "object", + properties: { + color: { + type: "string", + }, + border: { + type: ["string", "null"], + }, + }, + required: ["color", "border"], + additionalProperties: false, + }, + }, + required: ["style"], + additionalProperties: false, + }, + }, + }, + }, + }, + operation: "toRequest", + }, + }, { contract: "WireConformance", operation: "toRequest", diff --git a/runtime/typescript/packages/core/tests/spec-vectors.test.ts b/runtime/typescript/packages/core/tests/spec-vectors.test.ts index 137c9f467..157457114 100644 --- a/runtime/typescript/packages/core/tests/spec-vectors.test.ts +++ b/runtime/typescript/packages/core/tests/spec-vectors.test.ts @@ -772,12 +772,7 @@ function buildAgentFromWireInput(input: any, name?: string): Agent { // Build tools if (input.tools && input.tools.length > 0) { agent.tools = input.tools.map((t: any) => { - const params = (t.parameters ?? []).map((p: any) => new Property({ - name: p.name, - kind: p.kind, - required: p.required, - description: p.description, - })); + const params = (t.parameters ?? []).map((p: any) => Property.load(p)); const bindings = Object.entries(t.bindings ?? {}).map(([bname, bval]: [string, any]) => new Binding({ name: bname, input: typeof bval === "object" ? bval.input : String(bval) }), ); @@ -794,12 +789,7 @@ function buildAgentFromWireInput(input: any, name?: string): Agent { // Build outputs if (input.outputs && input.outputs.length > 0) { - agent.outputs = input.outputs.map((o: any) => new Property({ - name: o.name, - kind: o.kind, - required: o.required, - description: o.description, - })); + agent.outputs = input.outputs.map((o: any) => Property.load(o)); } return agent; diff --git a/schema/model/conformance/vectors/wire.tsp b/schema/model/conformance/vectors/wire.tsp index 0fa999b41..5120c501e 100644 --- a/schema/model/conformance/vectors/wire.tsp +++ b/schema/model/conformance/vectors/wire.tsp @@ -627,6 +627,96 @@ const WireVectors = """ } } }, + { + "name": "structured_output_nested_optional", + "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", + "stage": "wire", + "provider": "openai", + "targetApi": "chat", + "input": { + "provider": "openai", + "apiType": "chat", + "model_id": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": [ + { + "kind": "text", + "value": "Choose a visual style." + } + ] + } + ], + "tools": [], + "options": {}, + "outputs": [ + { + "name": "style", + "kind": "object", + "required": true, + "properties": [ + { + "name": "color", + "kind": "string", + "required": true + }, + { + "name": "border", + "kind": "string", + "required": false + } + ] + } + ] + }, + "expected": { + "request_body": { + "model": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": "Choose a visual style." + } + ], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "structured_output", + "strict": true, + "schema": { + "type": "object", + "properties": { + "style": { + "type": "object", + "properties": { + "color": { + "type": "string" + }, + "border": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "color", + "border" + ], + "additionalProperties": false + } + }, + "required": [ + "style" + ], + "additionalProperties": false + } + } + } + } + } + }, { "name": "kind_to_json_type_mapping", "description": "§7.1.4 — All Property kind values mapped to JSON Schema types: string→string, integer→integer, float→number, boolean→boolean, array→array, object→object.", diff --git a/schema/tsp-output/.typra-generated/vectors.json b/schema/tsp-output/.typra-generated/vectors.json index 9e1fe670a..4c0fc12c7 100644 --- a/schema/tsp-output/.typra-generated/vectors.json +++ b/schema/tsp-output/.typra-generated/vectors.json @@ -9633,6 +9633,106 @@ "operation": "toRequest" } }, + { + "contract": "WireConformance", + "operation": "toRequest", + "params": { + "agent": "Agent", + "messages": "Message[]" + }, + "returns": "unknown", + "vector": { + "name": "structured_output_nested_optional", + "description": "§7.1.4/§7.1.6 — OpenAI strict mode recursively requires every object property and represents optional fields as nullable.", + "stage": "wire", + "provider": "openai", + "targetApi": "chat", + "input": { + "provider": "openai", + "apiType": "chat", + "model_id": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": [ + { + "kind": "text", + "value": "Choose a visual style." + } + ] + } + ], + "tools": [], + "options": {}, + "outputs": [ + { + "name": "style", + "kind": "object", + "required": true, + "properties": [ + { + "name": "color", + "kind": "string", + "required": true + }, + { + "name": "border", + "kind": "string", + "required": false + } + ] + } + ] + }, + "expected": { + "request_body": { + "model": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": "Choose a visual style." + } + ], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "structured_output", + "strict": true, + "schema": { + "type": "object", + "properties": { + "style": { + "type": "object", + "properties": { + "color": { + "type": "string" + }, + "border": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "color", + "border" + ], + "additionalProperties": false + } + }, + "required": [ + "style" + ], + "additionalProperties": false + } + } + } + } + }, + "operation": "toRequest" + } + }, { "contract": "WireConformance", "operation": "toRequest", diff --git a/spec/spec.md b/spec/spec.md index 67c554f69..4b6ef9635 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -1496,17 +1496,18 @@ A `list[Property]` (used for `inputs`, `outputs`, and `FunctionTool.parameters`) MUST be converted to a JSON Schema object for wire transmission: ``` -function schema_to_wire(properties: list[Property]) → dict: +function schema_to_wire(properties: list[Property], strict: bool = false) → dict: schema = { type: "object", properties: {}, required: [] } for prop in properties: - prop_schema = property_to_json_schema(prop) + optional = strict and not prop.required + prop_schema = property_to_json_schema(prop, strict, optional) if prop.description: prop_schema.description = prop.description if prop.enumValues: prop_schema.enum = prop.enumValues schema.properties[prop.name] = prop_schema - if prop.required: + if strict or prop.required: schema.required.append(prop.name) if schema.required is empty: @@ -1516,9 +1517,13 @@ function schema_to_wire(properties: list[Property]) → dict: ``` `property_to_json_schema` MUST recursively convert array items and object -properties. Object `required` arrays MUST contain only children whose -`Property.required` is true. For a concrete property with `nullable: true`, -the JSON Schema `type` MUST include both the concrete type and `"null"`. +properties. Outside provider strict mode, object `required` arrays MUST contain +only children whose `Property.required` is true. In OpenAI strict mode, this +rule MUST apply recursively: every property of every object MUST appear in that +object's `required` array, and a property whose `Property.required` is not true +MUST be represented as nullable. For a concrete property, nullability is +represented by a JSON Schema `type` containing both the concrete type and +`"null"`. `kind: "union"` represents a portable union property. Exactly one of its `oneOf` and `anyOf` arrays MUST be nonempty; implementations MUST reject diff --git a/web/src/content/docs/specification/wire-format.mdx b/web/src/content/docs/specification/wire-format.mdx index 65418b715..f36bf6fd1 100644 --- a/web/src/content/docs/specification/wire-format.mdx +++ b/web/src/content/docs/specification/wire-format.mdx @@ -179,17 +179,18 @@ A `list[Property]` (used for `inputs`, `outputs`, and `FunctionTool.parameters`) MUST be converted to a JSON Schema object for wire transmission: ``` -function schema_to_wire(properties: list[Property]) → dict: +function schema_to_wire(properties: list[Property], strict: bool = false) → dict: schema = { type: "object", properties: {}, required: [] } for prop in properties: - prop_schema = { type: map_kind_to_json_type(prop.kind) } + optional = strict and not prop.required + prop_schema = property_to_json_schema(prop, strict, optional) if prop.description: prop_schema.description = prop.description if prop.enumValues: prop_schema.enum = prop.enumValues schema.properties[prop.name] = prop_schema - if prop.required: + if strict or prop.required: schema.required.append(prop.name) if schema.required is empty: @@ -198,6 +199,13 @@ function schema_to_wire(properties: list[Property]) → dict: return schema ``` +`property_to_json_schema` MUST recurse through array items and object +properties. Outside provider strict mode, each object lists only explicitly +required children in `required`. In OpenAI strict mode, every property of every +object MUST appear in that object's `required` array. A property that is +optional in the Prompty model MUST instead be represented as nullable in the +strict wire schema. + **Kind → JSON Schema type mapping.** Implementations MUST use this table: | Property `kind` | JSON Schema `type` |