From e2faacc61a1399789b364e91ed360a6e70d9f951 Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Sun, 25 Oct 2026 08:19:20 -0300 Subject: [PATCH 1/2] fix: drop temperature when LiteLLM rejects it for newer Anthropic models LiteLLM 1.96 rejects temperature=0 client-side for claude-opus-5-5, claude-sonnet-5-5 and claude-fable-5-1 with an UnsupportedParamsError ("does not support temperature=0"). The existing fallback only matched Anthropic's "`temperature` is deprecated" message, so every Anthropic test failed with a BadRequestError. Match both messages, and remember the fallback per model so later calls skip the doomed first request. Co-Authored-By: Claude Opus 5.5 --- python/tests/test_models.py | 43 ++++++++++++++++++++++++++++++++++++- python/wp_bench/models.py | 18 +++++++++++----- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 3b291b8..721c434 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -2,7 +2,7 @@ from types import SimpleNamespace -from litellm.exceptions import BadRequestError +from litellm.exceptions import BadRequestError, UnsupportedParamsError import wp_bench.models as models_module from wp_bench.config import ModelConfig @@ -33,6 +33,47 @@ def fake_completion(**kwargs): assert "temperature" not in calls[1] +def test_generate_drops_temperature_litellm_rejects_and_remembers_it(monkeypatch) -> None: + calls: list[dict] = [] + + def fake_completion(**kwargs): + calls.append(kwargs) + if "temperature" in kwargs: + raise UnsupportedParamsError( + message="claude-opus-5-5 does not support temperature=0. Only temperature=1 is supported.", + model="anthropic/claude-opus-5-5", + llm_provider="anthropic", + ) + return SimpleNamespace(choices=[SimpleNamespace(message={"content": "ok"})]) + + monkeypatch.setattr(models_module, "completion", fake_completion) + + model = ModelInterface(ModelConfig(name="anthropic/claude-opus-5-5")) + first = model.generate_with_metadata("hello") + second = model.generate_with_metadata("again") + + assert first.text == "ok" and second.text == "ok" + assert first.temperature_fallback and second.temperature_fallback + # Only the first call pays for the rejection. + assert ["temperature" in call for call in calls] == [True, False, False] + + +def test_generate_reraises_unrelated_bad_request(monkeypatch) -> None: + def fake_completion(**kwargs): + raise BadRequestError(message="prompt is too long", model="m", llm_provider="anthropic") + + monkeypatch.setattr(models_module, "completion", fake_completion) + + model = ModelInterface(ModelConfig(name="anthropic/claude-opus-5-5")) + try: + model.generate("hello") + except BadRequestError: + pass + else: + raise AssertionError("expected BadRequestError") + assert "temperature" in model._completion_kwargs("x") + + def test_generate_normalizes_none_content_to_empty_string(monkeypatch) -> None: def fake_completion(**kwargs): return SimpleNamespace(choices=[SimpleNamespace(message={"content": None})]) diff --git a/python/wp_bench/models.py b/python/wp_bench/models.py index 31a1718..7780594 100644 --- a/python/wp_bench/models.py +++ b/python/wp_bench/models.py @@ -77,6 +77,9 @@ def __init__(self, config: ModelConfig, system_prompt: str | None = None): #: Variant state (e.g. injected skill content), deliberately not part #: of ModelConfig so it never lands in serialized model config blocks. self.system_prompt = system_prompt + #: Set once the provider rejects ``temperature`` for this model, so + #: later calls skip it instead of failing first every time. + self._omit_temperature = False def generate(self, prompt: str) -> str: """Generate a completion for the given prompt (text only). @@ -99,7 +102,7 @@ def generate_with_metadata(self, prompt: str) -> ModelGeneration: kwargs = self._completion_kwargs(prompt) started = time.perf_counter() attempt_count = 0 - temperature_fallback = False + temperature_fallback = self._omit_temperature def _should_retry(error: BaseException) -> bool: if not self._retry_enabled_for(error): @@ -129,9 +132,10 @@ def _before(retry_state: RetryCallState) -> None: # temperature case which is fixable by dropping the # parameter. That fallback is intentional behavior, # not a retry, and is flagged separately. - if not _is_deprecated_temperature_error(error) or "temperature" not in kwargs: + if not _is_unsupported_temperature_error(error) or "temperature" not in kwargs: raise kwargs.pop("temperature") + self._omit_temperature = True temperature_fallback = True response = completion(**kwargs) @@ -179,12 +183,16 @@ def _completion_kwargs(self, prompt: str) -> dict[str, Any]: "top_p": self.config.top_p, "timeout": self.config.request_timeout, } - kwargs["temperature"] = self.config.temperature + if not self._omit_temperature: + kwargs["temperature"] = self.config.temperature return kwargs -def _is_deprecated_temperature_error(error: BadRequestError) -> bool: - return "`temperature` is deprecated" in str(error) +def _is_unsupported_temperature_error(error: BadRequestError) -> bool: + # Anthropic's API says the parameter is deprecated; LiteLLM rejects it + # client-side for newer models with an UnsupportedParamsError. + message = str(error) + return "`temperature` is deprecated" in message or "does not support temperature" in message def _extract_usage(response: Any) -> dict[str, int | None]: From 2fc042fb6ebee5ec826ab665d2a3dcf994060ab6 Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Sun, 25 Oct 2026 08:49:54 -0300 Subject: [PATCH 2/2] fix: also match LiteLLM's gpt-5 / o-series temperature rejection LiteLLM words the same client-side rejection as "don't support temperature=" for gpt-5 and o-series models. Match both wordings, document what temperature_fallback now means, and tidy the tests. Co-Authored-By: Claude Opus 5.5 --- python/tests/test_models.py | 29 +++++++++++++++++++++++------ python/wp_bench/models.py | 14 +++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 721c434..07fcbb5 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -2,6 +2,7 @@ from types import SimpleNamespace +import pytest from litellm.exceptions import BadRequestError, UnsupportedParamsError import wp_bench.models as models_module @@ -54,10 +55,30 @@ def fake_completion(**kwargs): assert first.text == "ok" and second.text == "ok" assert first.temperature_fallback and second.temperature_fallback - # Only the first call pays for the rejection. assert ["temperature" in call for call in calls] == [True, False, False] +def test_generate_drops_temperature_for_openai_wording(monkeypatch) -> None: + calls: list[dict] = [] + + def fake_completion(**kwargs): + calls.append(kwargs) + if "temperature" in kwargs: + raise UnsupportedParamsError( + message="gpt-5 models (including gpt-5-codex) don't support temperature=0.0. Only temperature=1 is supported.", + model="gpt-5", + llm_provider="openai", + ) + return SimpleNamespace(choices=[SimpleNamespace(message={"content": "ok"})]) + + monkeypatch.setattr(models_module, "completion", fake_completion) + + result = ModelInterface(ModelConfig(name="gpt-5")).generate_with_metadata("hello") + + assert result.text == "ok" and result.temperature_fallback + assert "temperature" not in calls[-1] + + def test_generate_reraises_unrelated_bad_request(monkeypatch) -> None: def fake_completion(**kwargs): raise BadRequestError(message="prompt is too long", model="m", llm_provider="anthropic") @@ -65,12 +86,8 @@ def fake_completion(**kwargs): monkeypatch.setattr(models_module, "completion", fake_completion) model = ModelInterface(ModelConfig(name="anthropic/claude-opus-5-5")) - try: + with pytest.raises(BadRequestError): model.generate("hello") - except BadRequestError: - pass - else: - raise AssertionError("expected BadRequestError") assert "temperature" in model._completion_kwargs("x") diff --git a/python/wp_bench/models.py b/python/wp_bench/models.py index 7780594..cf6583e 100644 --- a/python/wp_bench/models.py +++ b/python/wp_bench/models.py @@ -52,6 +52,8 @@ class ModelGeneration: retry_count: int latency_ms: float provider_response_id: str | None + #: True when this call went out without ``temperature`` because the + #: model rejects it (see ``_is_unsupported_temperature_error``). temperature_fallback: bool = False prompt_tokens: int | None = None completion_tokens: int | None = None @@ -128,8 +130,8 @@ def _before(retry_state: RetryCallState) -> None: try: response = completion(**kwargs) except BadRequestError as error: - # Deterministic error, except the known deprecated- - # temperature case which is fixable by dropping the + # Deterministic error, except a model that rejects + # temperature, which is fixable by dropping the # parameter. That fallback is intentional behavior, # not a retry, and is flagged separately. if not _is_unsupported_temperature_error(error) or "temperature" not in kwargs: @@ -189,10 +191,12 @@ def _completion_kwargs(self, prompt: str) -> dict[str, Any]: def _is_unsupported_temperature_error(error: BadRequestError) -> bool: - # Anthropic's API says the parameter is deprecated; LiteLLM rejects it - # client-side for newer models with an UnsupportedParamsError. + # Anthropic's API says the parameter is deprecated. LiteLLM rejects it + # client-side with an UnsupportedParamsError for newer Anthropic models + # ("does not support temperature=0") and for gpt-5 / o-series models + # ("don't support temperature=0"). message = str(error) - return "`temperature` is deprecated" in message or "does not support temperature" in message + return "`temperature` is deprecated" in message or "support temperature=" in message def _extract_usage(response: Any) -> dict[str, int | None]: