diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 3b291b8..07fcbb5 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -2,7 +2,8 @@ from types import SimpleNamespace -from litellm.exceptions import BadRequestError +import pytest +from litellm.exceptions import BadRequestError, UnsupportedParamsError import wp_bench.models as models_module from wp_bench.config import ModelConfig @@ -33,6 +34,63 @@ def fake_completion(**kwargs): assert "temperature" not in calls[1] +def test_generate_drops_temperature_litellm_rejects_and_remembers_it(monkeypatch) -> None: + calls: list[dict] = [] + + def fake_completion(**kwargs): + calls.append(kwargs) + if "temperature" in kwargs: + raise UnsupportedParamsError( + message="claude-opus-5-5 does not support temperature=0. Only temperature=1 is supported.", + model="anthropic/claude-opus-5-5", + llm_provider="anthropic", + ) + return SimpleNamespace(choices=[SimpleNamespace(message={"content": "ok"})]) + + monkeypatch.setattr(models_module, "completion", fake_completion) + + model = ModelInterface(ModelConfig(name="anthropic/claude-opus-5-5")) + first = model.generate_with_metadata("hello") + second = model.generate_with_metadata("again") + + assert first.text == "ok" and second.text == "ok" + assert first.temperature_fallback and second.temperature_fallback + assert ["temperature" in call for call in calls] == [True, False, False] + + +def test_generate_drops_temperature_for_openai_wording(monkeypatch) -> None: + calls: list[dict] = [] + + def fake_completion(**kwargs): + calls.append(kwargs) + if "temperature" in kwargs: + raise UnsupportedParamsError( + message="gpt-5 models (including gpt-5-codex) don't support temperature=0.0. Only temperature=1 is supported.", + model="gpt-5", + llm_provider="openai", + ) + return SimpleNamespace(choices=[SimpleNamespace(message={"content": "ok"})]) + + monkeypatch.setattr(models_module, "completion", fake_completion) + + result = ModelInterface(ModelConfig(name="gpt-5")).generate_with_metadata("hello") + + assert result.text == "ok" and result.temperature_fallback + assert "temperature" not in calls[-1] + + +def test_generate_reraises_unrelated_bad_request(monkeypatch) -> None: + def fake_completion(**kwargs): + raise BadRequestError(message="prompt is too long", model="m", llm_provider="anthropic") + + monkeypatch.setattr(models_module, "completion", fake_completion) + + model = ModelInterface(ModelConfig(name="anthropic/claude-opus-5-5")) + with pytest.raises(BadRequestError): + model.generate("hello") + assert "temperature" in model._completion_kwargs("x") + + def test_generate_normalizes_none_content_to_empty_string(monkeypatch) -> None: def fake_completion(**kwargs): return SimpleNamespace(choices=[SimpleNamespace(message={"content": None})]) diff --git a/python/wp_bench/models.py b/python/wp_bench/models.py index 31a1718..cf6583e 100644 --- a/python/wp_bench/models.py +++ b/python/wp_bench/models.py @@ -52,6 +52,8 @@ class ModelGeneration: retry_count: int latency_ms: float provider_response_id: str | None + #: True when this call went out without ``temperature`` because the + #: model rejects it (see ``_is_unsupported_temperature_error``). temperature_fallback: bool = False prompt_tokens: int | None = None completion_tokens: int | None = None @@ -77,6 +79,9 @@ def __init__(self, config: ModelConfig, system_prompt: str | None = None): #: Variant state (e.g. injected skill content), deliberately not part #: of ModelConfig so it never lands in serialized model config blocks. self.system_prompt = system_prompt + #: Set once the provider rejects ``temperature`` for this model, so + #: later calls skip it instead of failing first every time. + self._omit_temperature = False def generate(self, prompt: str) -> str: """Generate a completion for the given prompt (text only). @@ -99,7 +104,7 @@ def generate_with_metadata(self, prompt: str) -> ModelGeneration: kwargs = self._completion_kwargs(prompt) started = time.perf_counter() attempt_count = 0 - temperature_fallback = False + temperature_fallback = self._omit_temperature def _should_retry(error: BaseException) -> bool: if not self._retry_enabled_for(error): @@ -125,13 +130,14 @@ def _before(retry_state: RetryCallState) -> None: try: response = completion(**kwargs) except BadRequestError as error: - # Deterministic error, except the known deprecated- - # temperature case which is fixable by dropping the + # Deterministic error, except a model that rejects + # temperature, which is fixable by dropping the # parameter. That fallback is intentional behavior, # not a retry, and is flagged separately. - if not _is_deprecated_temperature_error(error) or "temperature" not in kwargs: + if not _is_unsupported_temperature_error(error) or "temperature" not in kwargs: raise kwargs.pop("temperature") + self._omit_temperature = True temperature_fallback = True response = completion(**kwargs) @@ -179,12 +185,18 @@ def _completion_kwargs(self, prompt: str) -> dict[str, Any]: "top_p": self.config.top_p, "timeout": self.config.request_timeout, } - kwargs["temperature"] = self.config.temperature + if not self._omit_temperature: + kwargs["temperature"] = self.config.temperature return kwargs -def _is_deprecated_temperature_error(error: BadRequestError) -> bool: - return "`temperature` is deprecated" in str(error) +def _is_unsupported_temperature_error(error: BadRequestError) -> bool: + # Anthropic's API says the parameter is deprecated. LiteLLM rejects it + # client-side with an UnsupportedParamsError for newer Anthropic models + # ("does not support temperature=0") and for gpt-5 / o-series models + # ("don't support temperature=0"). + message = str(error) + return "`temperature` is deprecated" in message or "support temperature=" in message def _extract_usage(response: Any) -> dict[str, int | None]: