From 7b66fcb8137500e2be3f978aa369a4e852be3e1d Mon Sep 17 00:00:00 2001 From: Isabel Wu <231155141+wuisabel-gif@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:14:43 -0700 Subject: [PATCH 1/3] Anthropic: support strict tool use via ToolInfo.options Honour ToolInfo.options["strict"] = True in tool_params_for_tool_info: set strict on the ToolParam, strip extended JSON Schema validation keywords (which strict schemas reject, matching the response_schema path), and add the structured-outputs-2025-11-13 beta header when any tool is strict. This is the only way to get schema-guaranteed tool calls alongside thinking, where tool_choice cannot force a tool call. Fixes #4432 Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 4 ++ src/inspect_ai/model/_providers/anthropic.py | 29 ++++++-- tests/model/providers/test_anthropic.py | 69 ++++++++++++++++++++ 3 files changed, 95 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c05fc38b6d..4ed4d7938c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Unreleased + +- Anthropic: Support strict tool use via `ToolInfo.options["strict"] = True` (sets `strict: true` on the tool definition, strips unsupported schema validation keywords, and adds the structured outputs beta header). (#4432) + ## 0.3.245 (08 July 2026) - Model API: New `openai-api-completions` provider for the legacy `/v1/completions` endpoint of any OpenAI-compatible server (raw prompts, no chat template); shares its implementation with `vllm-completions`. diff --git a/src/inspect_ai/model/_providers/anthropic.py b/src/inspect_ai/model/_providers/anthropic.py index 68c117d3d7..9b71c80c07 100644 --- a/src/inspect_ai/model/_providers/anthropic.py +++ b/src/inspect_ai/model/_providers/anthropic.py @@ -500,6 +500,8 @@ async def generate( tool.get("type", None) == "web_fetch_20250910" for tool in tools_param ): betas.append("web-fetch-2025-09-10") + if any(tool.get("strict", None) is True for tool in tools_param): + betas.append("structured-outputs-2025-11-13") # extra_body if len(extra_body) > 0 or self.extra_body is not None: @@ -1514,13 +1516,26 @@ def tool_params_for_tool_info( ) -> Sequence["ToolParamDef"]: # Use a native tool implementation when available. Otherwise, use the # standard tool implementation - return self.maybe_native_tool_params(tool, config) or [ - ToolParam( - name=tool.name, - description=tool.description, - input_schema=json_schema_dump(tool.parameters), - ) - ] + native_params = self.maybe_native_tool_params(tool, config) + if native_params: + return native_params + + # strict tool use guarantees schema-conformant tool calls (and is the + # only way to do so alongside thinking, where tool_choice can't force + # a tool call). strict schemas reject extended validation keywords, + # so strip them as the response_schema path does. + strict = (tool.options or {}).get("strict") is True + param = ToolParam( + name=tool.name, + description=tool.description, + input_schema=json_schema_dump( + tool.parameters, + exclude=JSON_SCHEMA_EXTENDED_FIELDS if strict else None, + ), + ) + if strict: + param["strict"] = True + return [param] def mcp_server_param( self, mcp_server: MCPServerConfigHTTP diff --git a/tests/model/providers/test_anthropic.py b/tests/model/providers/test_anthropic.py index 017abac0e3..333813e38c 100644 --- a/tests/model/providers/test_anthropic.py +++ b/tests/model/providers/test_anthropic.py @@ -1372,3 +1372,72 @@ def test_anthropic_max_tokens_xhigh_max_floor( api = AnthropicAPI(model_name=model_name, api_key="test-key") config = GenerateConfig(**config_kwargs) assert api.max_tokens_for_config(config) == expected + + +def _emit_tool(strict: bool) -> ToolInfo: + from inspect_ai.tool import ToolParams + from inspect_ai.util import JSONSchema + + return ToolInfo( + name="emit", + description="emit a value", + parameters=ToolParams( + properties={"x": JSONSchema(type="integer", minimum=0, maximum=10)}, + required=["x"], + ), + options={"strict": True} if strict else None, + ) + + +def test_anthropic_tool_param_not_strict_by_default() -> None: + api = AnthropicAPI(model_name="claude-sonnet-4-6", api_key="test-key") + (param,) = api.tool_params_for_tool_info(_emit_tool(strict=False), GenerateConfig()) + assert "strict" not in param + assert param["input_schema"]["properties"]["x"]["minimum"] == 0 + + +def test_anthropic_strict_tool_param() -> None: + """ToolInfo.options["strict"] sets strict and strips extended schema fields.""" + api = AnthropicAPI(model_name="claude-sonnet-4-6", api_key="test-key") + (param,) = api.tool_params_for_tool_info(_emit_tool(strict=True), GenerateConfig()) + assert param["strict"] is True + x_schema = param["input_schema"]["properties"]["x"] + assert "minimum" not in x_schema + assert "maximum" not in x_schema + + +@pytest.mark.anyio +@pytest.mark.parametrize("strict", [True, False]) +async def test_anthropic_strict_tool_structured_outputs_beta(strict: bool) -> None: + """Strict tools add the structured outputs beta header (and only then).""" + api = AnthropicAPI(model_name="claude-sonnet-4-6", api_key="test-key") + + captured: dict[str, Any] = {} + + from inspect_ai.model._model_output import ModelOutput + + async def fake_perform( + request: dict[str, Any], + streaming: bool, + tools: list[Any], + config: GenerateConfig, + pending_tool_uses: Any = None, + pending_mcp_tool_uses: Any = None, + span_recorder: Any = None, + ) -> tuple[dict[str, Any], ModelOutput]: + captured.update(request) + return {}, ModelOutput.from_content( + model=api.service_model_name(), content="ok" + ) + + api._perform_request_and_continuations = fake_perform # type: ignore[method-assign] + + await api.generate( + input=[ChatMessageUser(content="call emit with x=1")], + tools=[_emit_tool(strict=strict)], + tool_choice="auto", + config=GenerateConfig(), + ) + + betas = captured.get("extra_headers", {}).get("anthropic-beta", "") + assert ("structured-outputs-2025-11-13" in betas) is strict From dbb165ac783463c510c0c1951af21ba8d0ebd253 Mon Sep 17 00:00:00 2001 From: Eric Patey Date: Mon, 13 Jul 2026 09:16:46 -0400 Subject: [PATCH 2/3] tests: failing test for nested additionalProperties under strict tool use Anthropic strict schemas require additionalProperties: false on all objects, nested included. The strict tool path only gets the top level (via the ToolParams default); nested object schemas omit the key. --- tests/model/providers/test_anthropic.py | 45 +++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/model/providers/test_anthropic.py b/tests/model/providers/test_anthropic.py index 333813e38c..19f96c3251 100644 --- a/tests/model/providers/test_anthropic.py +++ b/tests/model/providers/test_anthropic.py @@ -1396,6 +1396,51 @@ def test_anthropic_tool_param_not_strict_by_default() -> None: assert param["input_schema"]["properties"]["x"]["minimum"] == 0 +def test_anthropic_strict_tool_param_nested_additional_properties() -> None: + """Strict schemas require additionalProperties: false on all objects, nested included. + + https://platform.claude.com/docs/en/build-with-claude/structured-outputs + (schema limitations: "additionalProperties (must be set to false for objects)"). + The top level gets it from the ToolParams default; nested object schemas + must have it set explicitly, as set_additional_properties_false does for + the response_schema path. + """ + from inspect_ai.tool import ToolParams + from inspect_ai.util import JSONSchema + + api = AnthropicAPI(model_name="claude-sonnet-4-6", api_key="test-key") + tool = ToolInfo( + name="emit", + description="emit a value", + parameters=ToolParams( + properties={ + "obj": JSONSchema( + type="object", + properties={"y": JSONSchema(type="integer")}, + required=["y"], + ), + "items": JSONSchema( + type="array", + items=JSONSchema( + type="object", + properties={"z": JSONSchema(type="string")}, + required=["z"], + ), + ), + }, + required=["obj", "items"], + ), + options={"strict": True}, + ) + from anthropic.types import ToolParam as SDKToolParam + + (param,) = api.tool_params_for_tool_info(tool, GenerateConfig()) + schema = cast("dict[str, Any]", cast("SDKToolParam", param)["input_schema"]) + assert schema["additionalProperties"] is False + assert schema["properties"]["obj"]["additionalProperties"] is False + assert schema["properties"]["items"]["items"]["additionalProperties"] is False + + def test_anthropic_strict_tool_param() -> None: """ToolInfo.options["strict"] sets strict and strips extended schema fields.""" api = AnthropicAPI(model_name="claude-sonnet-4-6", api_key="test-key") From c99468c8612fbd44b57d5f22f3df005f0ede38ac Mon Sep 17 00:00:00 2001 From: Eric Patey Date: Mon, 13 Jul 2026 09:31:41 -0400 Subject: [PATCH 3/3] tests: live coverage for Anthropic strict tool use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flat-schema tests pass (strict accepted, extended keywords stripped, schema-conformant call with and without thinking). Nested-object tests fail with 400 'additionalProperties must be explicitly set to false' — live repro of the nested additionalProperties gap. --- tests/model/test_extended_json_schema.py | 131 ++++++++++++++++++++++- 1 file changed, 130 insertions(+), 1 deletion(-) diff --git a/tests/model/test_extended_json_schema.py b/tests/model/test_extended_json_schema.py index c9da5f2fb0..35b3f30547 100644 --- a/tests/model/test_extended_json_schema.py +++ b/tests/model/test_extended_json_schema.py @@ -22,7 +22,7 @@ ResponseSchema, get_model, ) -from inspect_ai.tool import ToolInfo, ToolParam, ToolParams +from inspect_ai.tool import ToolFunction, ToolInfo, ToolParam, ToolParams from inspect_ai.util import json_schema # Tool with extended JSON Schema constraint fields @@ -106,6 +106,135 @@ async def test_anthropic_extended_response_schema(): assert result.completion is not None +# CONSTRAINED_TOOL as a strict tool: flat schema, so only the extended +# constraint fields (which strict schemas reject) must be stripped for the +# API to accept it. +STRICT_FLAT_TOOL = CONSTRAINED_TOOL.model_copy(update={"options": {"strict": True}}) + + +def _assert_valid_flat_args(args: dict) -> None: + assert isinstance(args["username"], str) + assert isinstance(args["age"], int) + + +@skip_if_no_anthropic +async def test_anthropic_strict_flat_tool_schema(): + """The API accepts a strict tool with a flat schema and calls it conformantly.""" + model = get_model("anthropic/claude-sonnet-4-5") + result = await model.generate( + input=INPUT_TOOL, + tools=[STRICT_FLAT_TOOL], + tool_choice=ToolFunction(name="register_user"), + ) + tool_calls = result.message.tool_calls + assert tool_calls + _assert_valid_flat_args(tool_calls[0].arguments) + + +@skip_if_no_anthropic +async def test_anthropic_strict_flat_tool_schema_with_thinking(): + """Strict flat tool calls conform to schema when thinking is enabled. + + The motivating case for strict tool use (#4432): with thinking enabled + tool_choice is not sent, so strict is the only way to guarantee a + schema-conformant tool call. + """ + model = get_model("anthropic/claude-sonnet-4-6") + result = await model.generate( + input=INPUT_TOOL, + tools=[STRICT_FLAT_TOOL], + config=GenerateConfig(reasoning_tokens=1024, max_tokens=8192), + ) + tool_calls = result.message.tool_calls + assert tool_calls + _assert_valid_flat_args(tool_calls[0].arguments) + + +# Strict variant of CONSTRAINED_TOOL, with a nested object parameter. Under +# strict tool use the extended constraint fields must be stripped and every +# object schema (nested included) must carry additionalProperties: false, or +# the API rejects the request. +STRICT_TOOL = ToolInfo( + name="register_user", + description="Register a new user with a username, age, and address.", + parameters=ToolParams( + properties={ + "username": ToolParam( + type="string", + description="The username.", + pattern=r"^[a-zA-Z0-9_]+$", + minLength=3, + maxLength=20, + ), + "age": ToolParam( + type="integer", + description="The user's age.", + minimum=0, + maximum=150, + ), + "address": ToolParam( + type="object", + description="The user's address.", + properties={ + "city": ToolParam(type="string", description="City name."), + "postal_code": ToolParam(type="string", description="Postal code."), + }, + required=["city", "postal_code"], + ), + }, + required=["username", "age", "address"], + ), + options={"strict": True}, +) + +INPUT_STRICT_TOOL = [ + ChatMessageUser( + content="Register a user named alice who is 30 years old and lives " + "in Springfield, postal code 12345." + ) +] + + +def _assert_valid_register_user_args(args: dict) -> None: + assert isinstance(args["username"], str) + assert isinstance(args["age"], int) + assert isinstance(args["address"], dict) + assert {"city", "postal_code"} <= args["address"].keys() + + +@skip_if_no_anthropic +async def test_anthropic_strict_tool_schema(): + """The API accepts a strict tool schema (nested object included).""" + model = get_model("anthropic/claude-sonnet-4-5") + result = await model.generate( + input=INPUT_STRICT_TOOL, + tools=[STRICT_TOOL], + tool_choice=ToolFunction(name="register_user"), + ) + tool_calls = result.message.tool_calls + assert tool_calls + _assert_valid_register_user_args(tool_calls[0].arguments) + + +@skip_if_no_anthropic +async def test_anthropic_strict_tool_schema_with_thinking(): + """Strict tool calls conform to schema when thinking is enabled. + + The motivating case for strict tool use (#4432): with thinking enabled + tool_choice is not sent, so strict is the only way to guarantee a + schema-conformant tool call. + """ + model = get_model("anthropic/claude-sonnet-4-6") + result = await model.generate( + input=INPUT_STRICT_TOOL, + tools=[STRICT_TOOL], + config=GenerateConfig(reasoning_tokens=1024, max_tokens=8192), + ) + tool_calls = result.message.tool_calls + assert tool_calls + _assert_valid_register_user_args(tool_calls[0].arguments) + + # -- Google -------------------------------------------------------------------