Drop temperature when LiteLLM rejects it for newer Anthropic models

python/tests/test_models.py

@@ -2,7 +2,8 @@
 
 from types import SimpleNamespace
 
-from litellm.exceptions import BadRequestError
+import pytest
+from litellm.exceptions import BadRequestError, UnsupportedParamsError
 
 import wp_bench.models as models_module
 from wp_bench.config import ModelConfig
@@ -33,6 +34,63 @@ def fake_completion(**kwargs):
     assert "temperature" not in calls[1]
 
 
+def test_generate_drops_temperature_litellm_rejects_and_remembers_it(monkeypatch) -> None:
+    calls: list[dict] = []
+
+    def fake_completion(**kwargs):
+        calls.append(kwargs)
+        if "temperature" in kwargs:
+            raise UnsupportedParamsError(
+                message="claude-opus-5-5 does not support temperature=0. Only temperature=1 is supported.",
+                model="anthropic/claude-opus-5-5",
+                llm_provider="anthropic",
+            )
+        return SimpleNamespace(choices=[SimpleNamespace(message={"content": "ok"})])
+
+    monkeypatch.setattr(models_module, "completion", fake_completion)
+
+    model = ModelInterface(ModelConfig(name="anthropic/claude-opus-5-5"))
+    first = model.generate_with_metadata("hello")
+    second = model.generate_with_metadata("again")
+
+    assert first.text == "ok" and second.text == "ok"
+    assert first.temperature_fallback and second.temperature_fallback
+    assert ["temperature" in call for call in calls] == [True, False, False]
+
+
+def test_generate_drops_temperature_for_openai_wording(monkeypatch) -> None:
+    calls: list[dict] = []
+
+    def fake_completion(**kwargs):
+        calls.append(kwargs)
+        if "temperature" in kwargs:
+            raise UnsupportedParamsError(
+                message="gpt-5 models (including gpt-5-codex) don't support temperature=0.0. Only temperature=1 is supported.",
+                model="gpt-5",
+                llm_provider="openai",
+            )
+        return SimpleNamespace(choices=[SimpleNamespace(message={"content": "ok"})])
+
+    monkeypatch.setattr(models_module, "completion", fake_completion)
+
+    result = ModelInterface(ModelConfig(name="gpt-5")).generate_with_metadata("hello")
+
+    assert result.text == "ok" and result.temperature_fallback
+    assert "temperature" not in calls[-1]
+
+
+def test_generate_reraises_unrelated_bad_request(monkeypatch) -> None:
+    def fake_completion(**kwargs):
+        raise BadRequestError(message="prompt is too long", model="m", llm_provider="anthropic")
+
+    monkeypatch.setattr(models_module, "completion", fake_completion)
+
+    model = ModelInterface(ModelConfig(name="anthropic/claude-opus-5-5"))
+    with pytest.raises(BadRequestError):
+        model.generate("hello")
+    assert "temperature" in model._completion_kwargs("x")
+
+
 def test_generate_normalizes_none_content_to_empty_string(monkeypatch) -> None:
     def fake_completion(**kwargs):
         return SimpleNamespace(choices=[SimpleNamespace(message={"content": None})])

python/wp_bench/models.py

@@ -52,6 +52,8 @@ class ModelGeneration:
     retry_count: int
     latency_ms: float
     provider_response_id: str | None
+    #: True when this call went out without ``temperature`` because the
+    #: model rejects it (see ``_is_unsupported_temperature_error``).
     temperature_fallback: bool = False
     prompt_tokens: int | None = None
     completion_tokens: int | None = None
@@ -77,6 +79,9 @@ def __init__(self, config: ModelConfig, system_prompt: str | None = None):
         #: Variant state (e.g. injected skill content), deliberately not part
         #: of ModelConfig so it never lands in serialized model config blocks.
         self.system_prompt = system_prompt
+        #: Set once the provider rejects ``temperature`` for this model, so
+        #: later calls skip it instead of failing first every time.
+        self._omit_temperature = False
 
     def generate(self, prompt: str) -> str:
         """Generate a completion for the given prompt (text only).
@@ -99,7 +104,7 @@ def generate_with_metadata(self, prompt: str) -> ModelGeneration:
         kwargs = self._completion_kwargs(prompt)
         started = time.perf_counter()
         attempt_count = 0
-        temperature_fallback = False
+        temperature_fallback = self._omit_temperature
 
         def _should_retry(error: BaseException) -> bool:
             if not self._retry_enabled_for(error):
@@ -125,13 +130,14 @@ def _before(retry_state: RetryCallState) -> None:
                 try:
                     response = completion(**kwargs)
                 except BadRequestError as error:
-                    # Deterministic error, except the known deprecated-
-                    # temperature case which is fixable by dropping the
+                    # Deterministic error, except a model that rejects
+                    # temperature, which is fixable by dropping the
                     # parameter. That fallback is intentional behavior,
                     # not a retry, and is flagged separately.
-                    if not _is_deprecated_temperature_error(error) or "temperature" not in kwargs:
+                    if not _is_unsupported_temperature_error(error) or "temperature" not in kwargs:
                         raise
                     kwargs.pop("temperature")
+                    self._omit_temperature = True
                     temperature_fallback = True
                     response = completion(**kwargs)
 
@@ -179,12 +185,18 @@ def _completion_kwargs(self, prompt: str) -> dict[str, Any]:
             "top_p": self.config.top_p,
             "timeout": self.config.request_timeout,
         }
-        kwargs["temperature"] = self.config.temperature
+        if not self._omit_temperature:
+            kwargs["temperature"] = self.config.temperature
         return kwargs
 
 
-def _is_deprecated_temperature_error(error: BadRequestError) -> bool:
-    return "`temperature` is deprecated" in str(error)
+def _is_unsupported_temperature_error(error: BadRequestError) -> bool:
+    # Anthropic's API says the parameter is deprecated. LiteLLM rejects it
+    # client-side with an UnsupportedParamsError for newer Anthropic models
+    # ("does not support temperature=0") and for gpt-5 / o-series models
+    # ("don't support temperature=0").
+    message = str(error)
+    return "`temperature` is deprecated" in message or "support temperature=" in message
 
 
 def _extract_usage(response: Any) -> dict[str, int | None]: