Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions docs/configuration/llm_providers.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ Use cheap models for simple tasks, powerful models for complex tasks:
import lumen.ai as lmai

model_config = {
"default": {"model": "gpt-5.4-mini"}, # Cheap for most agents
"default": {"model": "gpt-5.6-luna"}, # Cheap for most agents
"sql": {"model": "gpt-4.1"}, # Powerful for SQL
"vega_lite": {"model": "gpt-4.1"}, # Powerful for charts
"deck_gl": {"model": "gpt-4.1"}, # Powerful for 3D maps
Expand Down Expand Up @@ -89,7 +89,7 @@ For installation and API key setup instructions, see the [Installation guide](..

| Provider | Default Model | Popular Models |
|----------|---------------|----------------|
| **OpenAI** | `gpt-5.4-mini` | `gpt-5.4`, `gpt-5.4-nano` |
| **OpenAI** | `gpt-5.6-luna` | `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano` |
| **Anthropic** | `claude-haiku-4-5` | `claude-sonnet-4-6`, `claude-opus-4-5` |
| **Google** | `gemini-3-flash-preview` | `gemini-3-pro-preview`, `gemini-2.5-flash`, `gemini-2.0-flash` |
| **Mistral** | `mistral-small-latest` | `mistral-large-latest`, `ministral-8b-latest` |
Expand All @@ -99,6 +99,8 @@ For installation and API key setup instructions, see the [Installation guide](..
!!! warning "Reasoning Models Not Suitable for Dialog"
Reasoning models like `gpt-5`, `o4-mini`, and `gemini-2.0-flash-thinking` are **significantly slower** than standard models. They are designed for single, complex queries that require deep thinking, not interactive chat interfaces. For dialog-based applications like Lumen, use standard models for better user experience.

The OpenAI default, `gpt-5.6-luna`, is a reasoning model. Chat completions rejects function tools while reasoning is active, so Lumen disables reasoning for it and keeps it as fast as a standard model. To use reasoning with function tools, switch to the responses API: `lmai.llm.OpenAI(api="responses")`.

### Local providers

| Provider | Default Model | Description |
Expand Down Expand Up @@ -461,7 +463,7 @@ Additional model types:

Different providers use different model string formats:

- **OpenAI**: `"gpt-5.4"`, `"gpt-5.4-mini"`, `"gpt-5.4-nano"`, `"gpt-5.4"`
- **OpenAI**: `"gpt-5.6-luna"`, `"gpt-5.4"`, `"gpt-5.4-mini"`, `"gpt-5.4-nano"`
- **Anthropic**: `"claude-sonnet-4-6"`, `"claude-haiku-4-5"`, `"claude-opus-4-5"`
- **OpenRouter**: `"openai/gpt-4o-mini"`, `"anthropic/claude-3.5-sonnet"`, `"google/gemini-2.5-flash"`
- **Google**: `"gemini-3-flash-preview"`, `"gemini-2.5-flash"`
Expand Down
61 changes: 60 additions & 1 deletion lumen/ai/llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,23 @@ class ImageResponse(BaseModel):
'kilo': 'Kilo',
}

# Request parameters an OpenAI-compatible model may reject, and the value to
# retry with; None omits the parameter entirely. Applied only after the API
# names the parameter in a 400, so new models need no entry here.
ADAPTIVE_KWARGS = {
"temperature": None,
"reasoning_effort": "none",
}


def find_bad_request(error: BaseException | None) -> openai.BadRequestError | None:
"""Find a provider 400 on an exception's cause chain, if there is one."""
while error is not None:
if isinstance(error, openai.BadRequestError):
return error
error = error.__cause__
return None


def get_available_llm() -> type[Llm] | None:
"""
Expand Down Expand Up @@ -1113,11 +1130,12 @@ class OpenAI(Llm, OpenAIMixin):
mode = param.Selector(default=Mode.TOOLS)

model_kwargs = param.Dict(default={
"default": {"model": "gpt-5.4-mini"}, # Use standard models, not reasoning models (gpt-5, o4-mini)
"default": {"model": "gpt-5.6-luna"}, # Runs with reasoning disabled; see _reasoning_models
"ui": {"model": "gpt-5.4-nano"},
})

select_models = param.List(default=[
"gpt-5.6-luna",
"gpt-5.2",
"gpt-5-mini",
"gpt-5-nano",
Expand All @@ -1130,6 +1148,12 @@ class OpenAI(Llm, OpenAIMixin):

_supports_logfire = True

def __init__(self, **params):
super().__init__(**params)
# Per-model request fixes learned from the API, e.g. gpt-5.6-luna
# rejecting a non-default temperature. See ADAPTIVE_KWARGS.
self._kwarg_fixes: dict[str, dict[str, Any]] = {}

@classmethod
def _resolve_openai_mode(cls, mode: Mode) -> Mode:
if mode in (Mode.RESPONSES_TOOLS, Mode.RESPONSES_TOOLS_WITH_INBUILT_TOOLS):
Expand Down Expand Up @@ -1402,7 +1426,42 @@ async def get_client(self, model_spec: str | dict, response_model: type[BaseMode
# Add timeout to the partial
return partial(client_callable.func, *client_callable.args, timeout=self.timeout, **client_callable.keywords)

def _apply_kwarg_fixes(self, model: str, kwargs: dict[str, Any]):
for key, value in self._kwarg_fixes.get(model, {}).items():
if value is None:
kwargs.pop(key, None)
else:
kwargs[key] = value

def _learn_kwarg_fix(self, model: str, error: openai.BadRequestError) -> bool:
"""
Record how to satisfy a model that rejected a request parameter,
returning whether anything new was learned. Explicit ``create_kwargs``
are never overridden, so a deliberate choice still surfaces its error.
"""
fixes = self._kwarg_fixes.setdefault(model, {})
if error.param not in ADAPTIVE_KWARGS or error.param in fixes or error.param in self.create_kwargs:
return False
fixes[error.param] = ADAPTIVE_KWARGS[error.param]
log_debug(f"Adapting to \033[96m{model!r}\033[0m: {error.param}={fixes[error.param]!r}")
return True

async def run_client(self, model_spec: str | dict, messages: list[Message] | list[dict[str, Any]], **kwargs):
model = self._get_model_kwargs(model_spec)["model"]
self._apply_kwarg_fixes(model, kwargs)
while True:
try:
return await self._send(model_spec, messages, **kwargs)
except Exception as e:
# instructor re-raises provider errors wrapped in its own
# retry exception, so the 400 is found on the cause chain.
error = find_bad_request(e)
# Each retry records one more parameter, so this terminates.
if error is None or not self._learn_kwarg_fix(model, error):
raise
self._apply_kwarg_fixes(model, kwargs)

async def _send(self, model_spec: str | dict, messages: list[Message] | list[dict[str, Any]], **kwargs):
if self.api == "chat_completions":
return await super().run_client(model_spec, messages, **kwargs)

Expand Down
145 changes: 145 additions & 0 deletions lumen/tests/ai/test_llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
from types import SimpleNamespace
from unittest.mock import MagicMock, patch

import httpx
import openai
import pytest

try:
Expand Down Expand Up @@ -401,6 +403,149 @@ async def fake_run_client(_model_spec, _messages, **kwargs):
assert final_call.get("max_retries") == 3


# ---------------------------------------------------------------------------
# Adaptive request kwargs
# ---------------------------------------------------------------------------

def _bad_request(param: str) -> openai.BadRequestError:
"""The 400 OpenAI returns when a model rejects a request parameter."""
request = httpx.Request("POST", "https://api.openai.com/v1/chat/completions")
return openai.BadRequestError(
"rejected",
response=httpx.Response(400, request=request),
body={"message": "rejected", "type": "invalid_request_error", "param": param},
)


def _capture_sends(monkeypatch, reject: list[str]) -> list[dict]:
"""Record every attempt, rejecting one parameter per entry in ``reject``."""
calls: list[dict] = []
rejections = list(reject)

async def fake_send(self, _model_spec, _messages, **kwargs):
calls.append(dict(kwargs))
if rejections:
raise _bad_request(rejections.pop(0))

monkeypatch.setattr(OpenAI, "_send", fake_send)
return calls


def test_openai_default_model_is_selectable():
"""The default has to appear in select_models: opening the settings dialog
rewrites model_kwargs to select_models[0] when the current model is missing
from the list, silently downgrading the default."""
default_model = OpenAI.param.model_kwargs.default["default"]["model"]
assert default_model == "gpt-5.6-luna"
assert default_model in OpenAI.param.select_models.default


async def test_rejected_temperature_is_dropped_and_retried(monkeypatch):
"""gpt-5.6-luna only accepts the default temperature, which Lumen would
otherwise send on every request."""
llm = OpenAI(model_kwargs={"default": {"model": "gpt-5.6-luna"}})
calls = _capture_sends(monkeypatch, reject=["temperature"])

# Start from _client_kwargs so this proves the shipped default is safe,
# rather than a temperature invented by the test.
await llm.run_client("default", [{"role": "user", "content": "hi"}], **llm._client_kwargs)

assert calls[0]["temperature"] == 0.25
assert "temperature" not in calls[1]


async def test_rejected_reasoning_effort_is_disabled_and_retried(monkeypatch):
"""Chat completions rejects function tools while reasoning is active; the
API's own error names reasoning_effort as the parameter to set."""
llm = OpenAI(model_kwargs={"default": {"model": "gpt-5.6-luna"}})
calls = _capture_sends(monkeypatch, reject=["temperature", "reasoning_effort"])

await llm.run_client("default", [{"role": "user", "content": "hi"}], **llm._client_kwargs)

assert "reasoning_effort" not in calls[1]
assert calls[2]["reasoning_effort"] == "none"


async def test_rejection_wrapped_by_instructor_is_still_adapted(monkeypatch):
"""instructor re-raises provider errors inside its own retry exception, so
the 400 has to be found on the cause chain rather than caught directly."""
llm = OpenAI(model_kwargs={"default": {"model": "gpt-5.6-luna"}})
calls: list[dict] = []
rejected = False

async def fake_send(self, _model_spec, _messages, **kwargs):
nonlocal rejected
calls.append(dict(kwargs))
if not rejected:
rejected = True
raise RuntimeError("instructor gave up") from _bad_request("temperature")

monkeypatch.setattr(OpenAI, "_send", fake_send)

await llm.run_client("default", [{"role": "user", "content": "hi"}], **llm._client_kwargs)

assert "temperature" not in calls[1]


async def test_learned_fixes_are_reused_without_another_rejection(monkeypatch):
"""The adaptation is paid once per model, not on every request."""
llm = OpenAI(model_kwargs={"default": {"model": "gpt-5.6-luna"}})
calls = _capture_sends(monkeypatch, reject=["temperature"])

await llm.run_client("default", [{"role": "user", "content": "hi"}], **llm._client_kwargs)
await llm.run_client("default", [{"role": "user", "content": "hi"}], **llm._client_kwargs)

assert len(calls) == 3
assert "temperature" not in calls[2]


@pytest.mark.parametrize(
("llm_factory", "model"),
[
(lambda model: OpenAI(model_kwargs={"default": {"model": model}}), "gpt-5.4-mini"),
(lambda model: OpenAI(model_kwargs={"default": {"model": model}}), "gpt-5.4-nano"),
(lambda model: Groq(api_key="k", model_kwargs={"default": {"model": model}}), "llama-3.3-70b-versatile"),
],
)
async def test_accepted_kwargs_are_left_alone(monkeypatch, llm_factory, model):
"""Models that accept sampling params must be left exactly as they were."""
llm = llm_factory(model)
calls = _capture_sends(monkeypatch, reject=[])

await llm.run_client("default", [{"role": "user", "content": "hi"}], **llm._client_kwargs)

assert len(calls) == 1
assert calls[0]["temperature"] == 0.25
assert "reasoning_effort" not in calls[0]


async def test_unknown_rejection_is_raised(monkeypatch):
"""Only parameters Lumen knows how to satisfy are retried; anything else
is the caller's problem and must not be swallowed."""
llm = OpenAI(model_kwargs={"default": {"model": "gpt-5.6-luna"}})
calls = _capture_sends(monkeypatch, reject=["messages"])

with pytest.raises(openai.BadRequestError):
await llm.run_client("default", [{"role": "user", "content": "hi"}], **llm._client_kwargs)

assert len(calls) == 1


async def test_explicit_create_kwargs_are_not_overridden(monkeypatch):
"""A deliberate create_kwargs choice surfaces its error rather than being
silently rewritten."""
llm = OpenAI(
model_kwargs={"default": {"model": "gpt-5.6-luna"}},
create_kwargs={"max_retries": 1, "reasoning_effort": "medium"},
)
calls = _capture_sends(monkeypatch, reject=["reasoning_effort"])

with pytest.raises(openai.BadRequestError):
await llm.run_client("default", [{"role": "user", "content": "hi"}], **llm._client_kwargs)

assert len(calls) == 1


# ---------------------------------------------------------------------------
# _normalize_multimodal_messages tests
# ---------------------------------------------------------------------------
Expand Down