diff --git a/.env.example b/.env.example index 45f745f..0766546 100644 --- a/.env.example +++ b/.env.example @@ -1,14 +1,54 @@ -OPENAI_API_KEY= -DEEPSEEK_API_KEY= +# Provider selection. CHULK_MODEL is required for openai-compatible, +# openrouter, anthropic, bedrock, and gemini. CHULK_LLM_PROVIDER=openai CHULK_MODEL= CHULK_LLM_FALLBACK_PROVIDERS= -CHULK_PERMISSION_PROFILE=workspace-write -CHULK_PROJECT_ROOT= -CHULK_RUNTIME_DIR=.chulk + +# OpenAI +OPENAI_API_KEY= + +# DeepSeek: CHULK_DEEPSEEK_API_KEY takes precedence over DEEPSEEK_API_KEY. +CHULK_DEEPSEEK_API_KEY= +DEEPSEEK_API_KEY= CHULK_DEEPSEEK_BASE_URL=https://api.deepseek.com + +# Local OpenAI-compatible server. The API key is optional. CHULK_LOCAL_BASE_URL=http://localhost:1234/v1 CHULK_LOCAL_API_KEY= + +# Hosted OpenAI-compatible endpoint. Both fields are required. +CHULK_OPENAI_COMPATIBLE_API_KEY= +CHULK_OPENAI_COMPATIBLE_BASE_URL= + +# OpenRouter: CHULK_OPENROUTER_API_KEY takes precedence over OPENROUTER_API_KEY. +CHULK_OPENROUTER_API_KEY= +OPENROUTER_API_KEY= +CHULK_OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 + +# Anthropic: CHULK_ANTHROPIC_API_KEY takes precedence over ANTHROPIC_API_KEY. +CHULK_ANTHROPIC_API_KEY= +ANTHROPIC_API_KEY= +CHULK_ANTHROPIC_BASE_URL= + +# Bedrock: set an explicit OpenAI-compatible endpoint; there is no default. +# Key precedence: CHULK_BEDROCK_API_KEY, BEDROCK_API_KEY, +# AWS_BEARER_TOKEN_BEDROCK. CHULK_BASE_URL is a legacy base-URL alias. +CHULK_BEDROCK_API_KEY= +BEDROCK_API_KEY= +AWS_BEARER_TOKEN_BEDROCK= +CHULK_BEDROCK_BASE_URL= +CHULK_BASE_URL= + +# Gemini: CHULK_GEMINI_API_KEY, then GEMINI_API_KEY, then GOOGLE_API_KEY. +CHULK_GEMINI_API_KEY= +GEMINI_API_KEY= +GOOGLE_API_KEY= +CHULK_GEMINI_BASE_URL= + +# Runtime and safety configuration. +CHULK_PERMISSION_PROFILE=workspace-write +CHULK_PROJECT_ROOT= +CHULK_RUNTIME_DIR=.chulk CHULK_HISTORY_LIMIT=20 CHULK_MAX_SKILLS_PER_TURN=3 CHULK_MAX_SKILL_CONTENT_CHARS=4000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0e85d8..f4f1e77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,10 +52,10 @@ jobs: - name: Install development dependencies run: | python -m pip install --upgrade pip - python -m pip install -e ".[dev,openai,mcp]" + python -m pip install -e ".[dev,providers,mcp]" - name: Typecheck package and external consumer - run: python -m mypy typing_tests + run: python -m mypy chulk typing_tests - name: Verify documentation contract run: python scripts/check_docs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c6c3e8c..d331956 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,52 @@ API is still pre-1.0. ### Added +- Added native Anthropic and Gemini providers plus OpenAI-compatible, + OpenRouter, and AWS Bedrock provider adapters. +- Added an all-provider `providers` installation extra while retaining the + individual `openai`, `anthropic`, and `gemini` extras. +- Added explicit provider profiles, shared connection binding, capability + metadata, and offline provider conformance tests with injected fake clients. +- Added native async model and structured-action requests for every built-in + provider transport, including async fallback chains and cancellation + propagation. +- Added deterministic offline evaluation helpers, versioned trace envelopes, + trace replay, and provider-free regression coverage. +- Added plan-step retry accounting and tool-attempt metadata to runtime state + and traces. - Added MIT licensing for repository and package consumers. - Added a security policy for responsible vulnerability reporting. - Added GitHub Actions CI for Python 3.11, 3.12, and 3.13. - Added clean-wheel install validation for public imports, packaged presets, bundled skills, runtime defaults, and example imports. +### Changed + +- Centralized validated model-action parsing inside the LLM boundary and kept + synchronous custom clients compatible through explicit adapter paths. +- Improved fallback behavior with shared context budgets, duplicate-request + prevention, and normalized timeout, connection, rate-limit, and server + failure classification. +- Extended CI type checking from the external SDK fixture to the package + implementation and retained the original custom-provider factory contract. +- Extended `chulk doctor` to validate requirements for every primary and + fallback provider. + +### Security + +- Reject credential-like content before durable memory storage. +- Deny model-facing reads of secret files, Git and trace data, SQLite state, + and private key material by default. +- Bound shell output while it is produced, terminate timed-out process groups, + and expose an explicit host-owned execution policy and containment gate. +- Sanitize provider base URLs in `--show-config` so user information, query + parameters, and fragments are never printed. + ### Documented +- Documented all provider names, optional dependencies, exact credential alias + precedence, explicit-model requirements, and Bedrock endpoint requirements. +- Clarified that provider tests are offline and live account validation remains + the responsibility of the application owner. - Documented that `chulkharness` is the install/package name and `chulk` is the import package and CLI command. diff --git a/README.md b/README.md index 5ace1f9..545c289 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,9 @@ python examples/00_sdk_quickstart.py ``` The first example is deterministic and needs no credentials. Hosted providers -are optional; for OpenAI use `python -m pip install "chulkharness[openai]"`. +are optional. Install every provider dependency with +`python -m pip install "chulkharness[providers]"`, or choose an individual +extra from the [provider guide](docs/providers.md). ## SDK @@ -49,7 +51,9 @@ under `.chulk/`. Read [configuration](docs/configuration.md), ## CLI -Install a provider extra, configure its credentials in a local `.env`, then run: +Install a provider extra, configure its credentials in a local `.env`, then run. +The exact provider names are `openai`, `deepseek`, `local`, +`openai-compatible`, `openrouter`, `anthropic`, `bedrock`, and `gemini`. ```bash chulk diff --git a/TODO.md b/TODO.md index 17c8304..a1feb1e 100644 --- a/TODO.md +++ b/TODO.md @@ -108,13 +108,13 @@ Module responsibilities: - [ ] `chulk/core/prompt_builder.py` - [ ] Compose prompts from base instructions, memory, skills, tools, and history. -- [ ] `chulk/llm/` - - [ ] Wrap the model provider API. - - [ ] Support OpenAI first. - - [ ] Hide provider-specific request and response details from the agent loop. - - [ ] Provide text completion and structured JSON completion helpers. - - [ ] Handle retries, timeouts, rate limits, and provider errors. - - [ ] Register providers through a provider registry and explicit capability metadata. +- [x] `chulk/llm/` + - [x] Wrap the model provider API. + - [x] Support OpenAI first. + - [x] Hide provider-specific request and response details from the agent loop. + - [x] Provide text completion and structured JSON completion helpers. + - [x] Handle retries, timeouts, rate limits, and provider errors. + - [x] Register providers through a provider registry and explicit capability metadata. - [x] `chulk/memory/` - [x] Manage short-term conversation history. @@ -263,10 +263,10 @@ Build a small provider wrapper before adding agent complexity. Future provider support: -- [ ] Add a minimal local/mock provider for tests. -- [ ] Add support for local LLMs later. +- [x] Add a minimal local/mock provider for tests. +- [x] Add support for local LLMs later. - [x] Add DeepSeek as an additional hosted provider. -- [ ] Add support for additional hosted providers later. +- [x] Add support for additional hosted providers later. - [x] Keep response normalization in one place. ## 5. Tool System @@ -1154,7 +1154,7 @@ Optional future directions: - [x] MCP-like tool interface. - [ ] Vector database. - [ ] Multi-agent mode. -- [ ] Local LLM support. +- [x] Local LLM support. - [ ] OpenTelemetry-style traces. - [ ] Permission UI. - [ ] Skill marketplace or folder installer. @@ -1164,7 +1164,7 @@ Optional future directions: - [ ] Docker development environment. - [ ] Configurable model/provider profiles. - [ ] Import/export memories. -- [ ] Agent evaluation scripts. +- [x] Agent evaluation scripts. - [ ] Benchmark prompts for regression testing. ## 17. Agent Harness Feature Backlog @@ -1248,8 +1248,11 @@ Feedback from the SDK/installability review changes the near-term priority. Befo - [x] Add a `Capabilities` config object or equivalent structured tool capability surface. - [ ] Add custom permission profiles in config. - [ ] Add workspace root allowlists. -- [ ] Add path deny rules for secrets, traces, SQLite stores, dependency folders, build artifacts, and credentials. +- [x] Add model-facing path deny rules for secrets, traces, Git metadata, SQLite/runtime state, private keys, and credentials. +- [ ] Extend explicit-read deny rules to dependency folders and build artifacts; directory search already excludes them. - [ ] Add command prefix rules: allow, prompt, and deny. +- [x] Add a host-owned shell execution policy boundary with explicit allow/deny and containment assertions. +- [x] Bound shell output during execution and terminate timed-out process groups. - [ ] Add network allow/deny domain rules. - [ ] Add trusted command catalog for low-risk read-only commands. - [ ] Add approval prompts with approve once, deny once, always allow, and always deny choices. @@ -1271,21 +1274,21 @@ Feedback from the SDK/installability review changes the near-term priority. Befo #### Phase E: Trace Productization - [ ] Treat traces as a public product surface, not only internal logs. -- [ ] Add a trace schema version to every JSONL event. -- [ ] Standardize trace event envelope fields: schema version, event/type, conversation id, turn id, timestamp, and payload. +- [x] Add a trace schema version to every JSONL event. +- [x] Standardize trace event envelope fields: schema version, event/type, conversation id, turn id, timestamp, and payload. - [x] Add a public trace reader: `Trace.from_jsonl(path)`. - [x] Expose typed trace events with event type and timestamp. - [x] Add `chulk trace inspect `. - [x] Add `chulk trace export --format html`. -- [ ] Add `chulk trace replay `. +- [x] Add `chulk trace replay `. - [x] Add a minimal HTML trace export. - [ ] Include user request, prompt context, loaded memories, loaded skills, tool calls, tool outputs, permission decisions, final answer, errors, token usage, and cost usage in trace exports. - [ ] Log available tools in traces. -- [ ] Log timing information in traces. -- [ ] Add `session_started` and `session_finished` trace events. +- [x] Log timing information in traces. +- [x] Add `session_started` and `session_finished` trace events. - [ ] Make it obvious why a tool was called where the model/action metadata supports it. - [ ] Make it obvious which memories were injected. -- [ ] Add trace redaction tests. +- [x] Add trace redaction tests. - [ ] Add artifact reader support for full truncated outputs saved under traces. - [ ] Add replay-friendly event coverage for regression tests. @@ -1300,7 +1303,9 @@ Feedback from the SDK/installability review changes the near-term priority. Befo - [x] Add tool-level retry policy. - [ ] Add tool testing helpers such as `assert_tool_schema(...)` and `invoke_tool(...)`. - [x] Continue supporting async tools through the public decorator. -- [ ] Add native async runtime later with async LLM calls, async tools, cancellation, timeouts, streaming events, concurrent read-only tools, and cleanup hooks. +- [x] Add native async LLM and structured-action calls for built-in providers and fallback chains. +- [x] Propagate native provider cancellation without wrapping it or advancing to a fallback provider. +- [ ] Complete the async runtime with concurrent read-only tools and async provider cleanup hooks. - [ ] Add memory namespaces for user and workspace isolation. - [x] Add explicit memory modes: off, read-only, manual read/write, and automatic read/write. - [x] Default SDK inferred-memory writes to off or manual review unless the developer opts in. @@ -1317,9 +1322,10 @@ Feedback from the SDK/installability review changes the near-term priority. Befo - [ ] Split docs by audience under `docs/`: index, quickstart, sdk, cli, tools, permissions, skills, memory, tracing, providers, mcp, safety, and release policy. - [ ] Add one end-to-end `examples/repo_review_bot/` killer example. - [ ] Include a sample trace export in the repo review example. -- [ ] Add fake provider or fake LLM support for examples and CI. -- [ ] Run examples with fake LLM in CI. -- [ ] Add a small eval harness with deterministic evals for tool calling, memory retrieval, skill selection, plan approval, and safety refusal. +- [x] Add fake provider or fake LLM support for examples and CI. +- [x] Run examples with fake LLM in CI. +- [x] Add a small deterministic, offline eval harness for agent outcomes, tool sequences, and trace sequences. +- [ ] Add bundled eval scenarios for memory retrieval, skill selection, plan approval, and safety refusal. - [ ] Add optional live-model evals separately from deterministic CI. - [ ] Improve plan mode as a public workflow primitive with richer `Plan` and `PlanStep` objects. - [ ] Add plan step fields for risk, tools, and acceptance criteria in the public object. @@ -1383,7 +1389,8 @@ Feedback from the SDK/installability review changes the near-term priority. Befo - [x] Add built-in permission profiles: read-only, workspace-write, trusted-local, and full-access. - [ ] Add custom permission profiles in config. - [ ] Add workspace root allowlists. -- [ ] Add path deny rules for secrets, traces, SQLite stores, dependency folders, and build artifacts. +- [x] Add default file-read deny rules for secrets, traces, Git metadata, SQLite/runtime state, and private keys. +- [ ] Extend explicit-read deny rules to dependency folders and build artifacts. - [ ] Add network allow/deny domain rules. - [ ] Add command prefix rules: allow, prompt, deny. - [ ] Add trusted command catalog for low-risk read-only commands. @@ -1589,7 +1596,7 @@ Feedback from the SDK/installability review changes the near-term priority. Befo - [ ] Add review against a base branch. - [ ] Add review of a selected commit. - [ ] Add custom review instructions. -- [ ] Add eval scripts for agent workflows. +- [x] Add eval scripts for agent workflows. - [ ] Add benchmark prompts for regression testing. - [ ] Add trace-based regression replay. - [ ] Add golden tests for tool-use loops. diff --git a/chulk/__init__.py b/chulk/__init__.py index e0c0417..32dd281 100644 --- a/chulk/__init__.py +++ b/chulk/__init__.py @@ -52,6 +52,9 @@ RunStartedPayload, RunStatus, SafetyError, + ShellExecutionDecision, + ShellExecutionPolicy, + ShellExecutionRequest, ToolExecutionError, ToolOutputPolicy, ToolRetryPolicy, @@ -129,6 +132,9 @@ "RunStartedPayload", "RunStatus", "SafetyError", + "ShellExecutionDecision", + "ShellExecutionPolicy", + "ShellExecutionRequest", "Skills", "Tool", "ToolContext", diff --git a/chulk/_sdk/config.py b/chulk/_sdk/config.py index b29680f..e931f3f 100644 --- a/chulk/_sdk/config.py +++ b/chulk/_sdk/config.py @@ -52,6 +52,16 @@ class AgentConfig: deepseek_base_url: str | None = None local_api_key: str | None = None local_base_url: str | None = None + openai_compatible_api_key: str | None = None + openai_compatible_base_url: str | None = None + openrouter_api_key: str | None = None + openrouter_base_url: str | None = None + anthropic_api_key: str | None = None + anthropic_base_url: str | None = None + bedrock_api_key: str | None = None + bedrock_base_url: str | None = None + gemini_api_key: str | None = None + gemini_base_url: str | None = None permission_profile: str | None = None store_path: str | Path | None = None traces_dir: str | Path | None = None @@ -80,8 +90,9 @@ def __post_init__(self) -> None: object.__setattr__(self, "llm_fallback_providers", tuple(self.llm_fallback_providers)) if self.memory_mode is not None: base = self.capabilities or Capabilities.read_only() - object.__setattr__(self, "capabilities", base.with_memory(self.memory_mode)) - object.__setattr__(self, "memory_mode", self.capabilities.memory) + capabilities = base.with_memory(self.memory_mode) + object.__setattr__(self, "capabilities", capabilities) + object.__setattr__(self, "memory_mode", capabilities.memory) def resolved_capabilities(self) -> Capabilities: """Return explicit capabilities or the safe SDK default.""" @@ -168,6 +179,94 @@ def local( values["local_base_url"] = base_url return cls.from_env(provider="local", model=model or DEFAULT_LOCAL_MODEL, **values) + @classmethod + def openai_compatible( + cls, + *, + model: str, + api_key: str | None = None, + base_url: str | None = None, + **kwargs: Any, + ) -> "AgentConfig": + """Create config for a hosted OpenAI-compatible endpoint.""" + values = dict(kwargs) + if api_key is not None: + values["openai_compatible_api_key"] = api_key + if base_url is not None: + values["openai_compatible_base_url"] = base_url + return cls.from_env(provider="openai-compatible", model=model, **values) + + @classmethod + def openrouter( + cls, + *, + model: str, + api_key: str | None = None, + base_url: str | None = None, + **kwargs: Any, + ) -> "AgentConfig": + """Create config for OpenRouter-backed agents.""" + values = dict(kwargs) + if api_key is not None: + values["openrouter_api_key"] = api_key + if base_url is not None: + values["openrouter_base_url"] = base_url + return cls.from_env(provider="openrouter", model=model, **values) + + @classmethod + def anthropic( + cls, + *, + model: str, + api_key: str | None = None, + base_url: str | None = None, + **kwargs: Any, + ) -> "AgentConfig": + """Create config for Anthropic-backed agents.""" + values = dict(kwargs) + if api_key is not None: + values["anthropic_api_key"] = api_key + if base_url is not None: + values["anthropic_base_url"] = base_url + return cls.from_env(provider="anthropic", model=model, **values) + + @classmethod + def bedrock( + cls, + *, + model: str, + api_key: str | None = None, + base_url: str | None = None, + **kwargs: Any, + ) -> "AgentConfig": + """Create config for an AWS Bedrock OpenAI-compatible endpoint.""" + values = dict(kwargs) + if api_key is not None: + values["bedrock_api_key"] = api_key + if base_url is not None: + values["bedrock_base_url"] = base_url + return cls.from_env(provider="bedrock", model=model, **values) + + @classmethod + def gemini( + cls, + *, + model: str, + api_key: str | None = None, + base_url: str | None = None, + **kwargs: Any, + ) -> "AgentConfig": + """Create config for Google Gemini-backed agents.""" + resolved_model = model.strip() + if not resolved_model: + raise ValueError("AgentConfig.gemini requires a non-empty model") + values = dict(kwargs) + if api_key is not None: + values["gemini_api_key"] = api_key + if base_url is not None: + values["gemini_base_url"] = base_url + return cls.from_env(provider="gemini", model=resolved_model, **values) + def to_config(self) -> Config: """Build the internal runtime config.""" env = dict(os.environ) @@ -182,6 +281,16 @@ def to_config(self) -> Config: _set_env(env, "CHULK_DEEPSEEK_BASE_URL", self.deepseek_base_url) _set_env(env, "CHULK_LOCAL_API_KEY", self.local_api_key) _set_env(env, "CHULK_LOCAL_BASE_URL", self.local_base_url) + _set_env(env, "CHULK_OPENAI_COMPATIBLE_API_KEY", self.openai_compatible_api_key) + _set_env(env, "CHULK_OPENAI_COMPATIBLE_BASE_URL", self.openai_compatible_base_url) + _set_env(env, "CHULK_OPENROUTER_API_KEY", self.openrouter_api_key) + _set_env(env, "CHULK_OPENROUTER_BASE_URL", self.openrouter_base_url) + _set_env(env, "CHULK_ANTHROPIC_API_KEY", self.anthropic_api_key) + _set_env(env, "CHULK_ANTHROPIC_BASE_URL", self.anthropic_base_url) + _set_env(env, "CHULK_BEDROCK_API_KEY", self.bedrock_api_key) + _set_env(env, "CHULK_BEDROCK_BASE_URL", self.bedrock_base_url) + _set_env(env, "CHULK_GEMINI_API_KEY", self.gemini_api_key) + _set_env(env, "CHULK_GEMINI_BASE_URL", self.gemini_base_url) _set_env(env, "CHULK_PERMISSION_PROFILE", self.permission_profile) _set_env(env, "CHULK_HISTORY_LIMIT", self.history_limit) _set_env(env, "CHULK_MAX_TOOL_CALLS_PER_TURN", self.max_tool_calls_per_turn) @@ -218,21 +327,25 @@ def to_config(self) -> Config: if self.skills_dir is not None else runtime_dir / "skills" ) - updates: dict[str, object] = { - "project_root": project_root, - "runtime_dir": runtime_dir, - "store_path": store_path, - "traces_dir": traces_dir, - "skills_dir": skills_dir, - "skills_dirs": _skills_dirs(skills_dir), - "mcp_config_path": config.mcp_config_path, - "permission_profile": permission_profile, - } + config = replace( + config, + project_root=project_root, + runtime_dir=runtime_dir, + store_path=store_path, + traces_dir=traces_dir, + skills_dir=skills_dir, + skills_dirs=_skills_dirs(skills_dir), + mcp_config_path=config.mcp_config_path, + permission_profile=permission_profile, + ) if self.mcp_servers is not None: - updates["mcp_servers"] = tuple(self.mcp_servers) + config = replace(config, mcp_servers=tuple(self.mcp_servers)) if self.llm_fallback_providers is not None: - updates["llm_fallback_providers"] = tuple(self.llm_fallback_providers) - return replace(config, **updates) + config = replace( + config, + llm_fallback_providers=tuple(self.llm_fallback_providers), + ) + return config class MCP: diff --git a/chulk/_sdk/error_mapping.py b/chulk/_sdk/error_mapping.py index 26b8af5..ae7ea2d 100644 --- a/chulk/_sdk/error_mapping.py +++ b/chulk/_sdk/error_mapping.py @@ -19,6 +19,7 @@ ) from chulk.llm.base import LLMConfigurationError, LLMError from chulk.mcp.config import MCPConfigError +from chulk.memory.security import MemorySecretError from chulk.tools.schema import ToolValidationError @@ -38,6 +39,8 @@ def map_public_error( return ConfigurationError(str(exc), details=details) if isinstance(exc, LLMError): return ProviderError(str(exc), details=details) + if isinstance(exc, MemorySecretError): + return SafetyError(str(exc), details=details) if isinstance(exc, PermissionError): return PermissionDeniedError(str(exc), details=details) if isinstance(exc, ToolValidationError) or getattr(exc, "tool_name", None): @@ -85,6 +88,9 @@ def _details( policy_name = getattr(exc, "policy_name", None) if policy_name: extensions["policy_name"] = policy_name + if isinstance(exc, LLMError): + extensions["error_code"] = exc.code + extensions["fallback_eligible"] = exc.fallback_eligible return ErrorDetails( provider=str(provider) if provider is not None else None, model=str(model) if model is not None else None, diff --git a/chulk/_sdk/events.py b/chulk/_sdk/events.py index c33863c..f275edd 100644 --- a/chulk/_sdk/events.py +++ b/chulk/_sdk/events.py @@ -60,7 +60,8 @@ def project_event(runtime: CoreAgent, event_type: str, payload: dict[str, Any]) extensions = {"internal_event": event_type} if event_type == TraceEvent.TURN_STARTED: - turn = payload.get("turn") if isinstance(payload.get("turn"), dict) else {} + turn_value = payload.get("turn") + turn = turn_value if isinstance(turn_value, dict) else {} return _event(EventName.RUN_STARTED, conversation_id, turn_id, RunStartedPayload(str(turn.get("user_message") or "")), extensions) if event_type == TraceEvent.MODEL_REQUEST_STARTED: return _event( @@ -107,7 +108,8 @@ def project_event(runtime: CoreAgent, event_type: str, payload: dict[str, Any]) extensions, ) if event_type in {TraceEvent.TOOL_PERMISSION_REQUESTED, TraceEvent.MCP_APPROVAL_REQUESTED}: - request = payload.get("request") if isinstance(payload.get("request"), dict) else {} + request_value = payload.get("request") + request = request_value if isinstance(request_value, dict) else {} return _event( EventName.PERMISSION_REQUESTED, conversation_id, @@ -120,7 +122,8 @@ def project_event(runtime: CoreAgent, event_type: str, payload: dict[str, Any]) extensions, ) if event_type in {TraceEvent.TOOL_PERMISSION_DECIDED, TraceEvent.MCP_APPROVAL_DECIDED}: - decision = payload.get("decision") if isinstance(payload.get("decision"), dict) else {} + decision_value = payload.get("decision") + decision = decision_value if isinstance(decision_value, dict) else {} return _event( EventName.PERMISSION_RESOLVED, conversation_id, @@ -179,6 +182,7 @@ def project_event(runtime: CoreAgent, event_type: str, payload: dict[str, Any]) def terminal_event(result: Any) -> AgentEvent: """Create the exact in-band terminal event for a generator run result.""" + payload: RunFailedPayload | RunCompletedPayload if getattr(result, "status", None) in {"failed", "blocked"}: errors = getattr(result, "errors", ()) message = errors[-1] if errors else getattr(result, "content", "The run failed.") diff --git a/chulk/_sdk/facade.py b/chulk/_sdk/facade.py index c745e16..6be5cec 100644 --- a/chulk/_sdk/facade.py +++ b/chulk/_sdk/facade.py @@ -27,9 +27,9 @@ from chulk.llm import LLMClient from chulk.events import AgentEvent, EventName from chulk.mcp import MCPServerConfig -from chulk.results import MemoryProposal +from chulk.results import MemoryProposal, RunStatus from chulk.runtime import create_agent as create_runtime_agent -from chulk.tools import ToolExecutionContext +from chulk.tools import ShellExecutionPolicy, ToolExecutionContext from chulk.tools.permissions import PermissionDecision, PermissionDecisionRecord, PermissionRequest @@ -241,7 +241,7 @@ def _run_result(self, content: str) -> RunResult: def _no_pending_plan_result(self, content: str) -> RunResult: return RunResult( content=content, - status="no_pending_plan", + status=RunStatus.NO_PENDING_PLAN, turn_id=None, conversation_id=self.conversation_id, trace_path=self.trace_path, @@ -342,8 +342,7 @@ async def run_result( return self.handle._run_result(content) async def plan(self, message: str) -> str: - self.handle._ensure_open() - return await asyncio.to_thread(self.handle.plan, message) + return (await self.plan_result(message)).content async def plan_result( self, @@ -353,7 +352,16 @@ async def plan_result( on_event: EventCallback | None = None, ) -> PlanResult: self.handle._ensure_open() - return await asyncio.to_thread(self.handle.plan_result, message, on_delta=on_delta, on_event=on_event) + previous_on_delta = self.handle._active_on_delta + previous_on_event = self.handle._active_on_event + self.handle._active_on_delta = on_delta + self.handle._active_on_event = on_event + try: + content = await self.runtime.run_planned_turn_async(message) + finally: + self.handle._active_on_delta = previous_on_delta + self.handle._active_on_event = previous_on_event + return self.handle._plan_result(content) async def approve(self) -> str: return (await self.approve_result()).content @@ -424,6 +432,8 @@ def __init__( capabilities: Capabilities | None = None, memory_mode: MemoryMode | str | None = None, deps: object | None = None, + shell_execution_policy: ShellExecutionPolicy | None = None, + require_shell_containment: bool = False, ) -> None: selected_capabilities = _selected_capabilities(config, capabilities, memory_mode) try: @@ -442,6 +452,8 @@ def __init__( redaction_fail_closed=redaction_fail_closed, capabilities=selected_capabilities, deps=deps, + shell_execution_policy=shell_execution_policy, + require_shell_containment=require_shell_containment, ) except Exception as exc: mapped = map_public_error(exc, config=config, operation="construct") @@ -789,6 +801,8 @@ def _build_handle( redaction_fail_closed: bool = False, capabilities: Capabilities | None = None, deps: object | None = None, + shell_execution_policy: ShellExecutionPolicy | None = None, + require_shell_containment: bool = False, ) -> AgentHandle: runtime_config = coerce_config(config) selected_tools = tools if tools is not None else (preset.tools if preset is not None else None) @@ -807,6 +821,8 @@ def _build_handle( redaction_fail_closed=redaction_fail_closed, capabilities=capabilities, deps=deps, + shell_execution_policy=shell_execution_policy, + require_shell_containment=require_shell_containment, ) return AgentHandle(runtime, on_event=on_event) diff --git a/chulk/_sdk/results.py b/chulk/_sdk/results.py index 321da0e..da31895 100644 --- a/chulk/_sdk/results.py +++ b/chulk/_sdk/results.py @@ -4,7 +4,7 @@ from collections.abc import Mapping from decimal import Decimal -from typing import Any +from typing import Any, SupportsIndex, SupportsInt, TypeAlias, cast from chulk.llm.usage import cost_snapshot_data, usage_snapshot_data from chulk.results import ( @@ -271,9 +271,12 @@ def _dict(value: object) -> dict[str, Any]: return _mapping(value) +_IntInput: TypeAlias = str | bytes | bytearray | SupportsInt | SupportsIndex + + def _int(value: object, *, default: int = 0) -> int: try: - return int(value) if value is not None else default + return int(cast(_IntInput, value)) if value is not None else default except (TypeError, ValueError): return default diff --git a/chulk/api.py b/chulk/api.py index 7c37982..629894a 100644 --- a/chulk/api.py +++ b/chulk/api.py @@ -61,7 +61,7 @@ ToolAttempt, Usage, ) -from chulk.tools import ToolContext +from chulk.tools import ShellExecutionDecision, ShellExecutionPolicy, ShellExecutionRequest, ToolContext __all__ = [ @@ -112,6 +112,9 @@ "RunStartedPayload", "RunStatus", "SafetyError", + "ShellExecutionDecision", + "ShellExecutionPolicy", + "ShellExecutionRequest", "ToolExecutionError", "ToolOutputPolicy", "ToolRetryPolicy", diff --git a/chulk/capabilities.py b/chulk/capabilities.py index 38ebd90..03ce9be 100644 --- a/chulk/capabilities.py +++ b/chulk/capabilities.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, replace from enum import StrEnum from types import MappingProxyType -from typing import Any +from typing import Any, cast class FileAccess(StrEnum): @@ -64,9 +64,9 @@ def with_memory(self, mode: MemoryMode | str) -> "Capabilities": def to_dict(self) -> dict[str, Any]: return { - "files": self.files.value, + "files": cast(FileAccess, self.files).value, "shell": self.shell, - "memory": self.memory.value, + "memory": cast(MemoryMode, self.memory).value, "network": self.network, "external_services": self.external_services, "utilities": self.utilities, diff --git a/chulk/cli/entrypoints.py b/chulk/cli/entrypoints.py index 01ba78b..abbbd03 100644 --- a/chulk/cli/entrypoints.py +++ b/chulk/cli/entrypoints.py @@ -11,9 +11,11 @@ export_trace_html, format_doctor_report, format_init_changes, + format_trace_replay, format_trace_summary, initialize_project, inspect_trace, + replay_trace, run_doctor, ) from chulk.core import Agent @@ -169,6 +171,12 @@ def run_trace_command( summary = inspect_trace(path) output_func(json_text(summary) if json_output else format_trace_summary(summary)) return EXIT_OK + if command == "replay": + replay = replay_trace(path) + output_func(json_text(replay) if json_output else format_trace_replay(replay)) + return EXIT_OK + if command != "export": + raise ValueError(f"Unknown trace command: {command}") destination = export_trace_html(path, output_path=output_path, force=force) payload = { "ok": True, diff --git a/chulk/cli/maintenance.py b/chulk/cli/maintenance.py index 74ff823..76b4272 100644 --- a/chulk/cli/maintenance.py +++ b/chulk/cli/maintenance.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import asdict, dataclass +from importlib.util import find_spec import json import os from pathlib import Path @@ -14,6 +15,18 @@ from chulk.tracing import Trace, TraceFormatError +_PROVIDER_SDK_REQUIREMENTS = { + "openai": ("openai", "openai", "openai"), + "deepseek": ("openai", "openai", "openai"), + "local": ("openai", "openai", "openai"), + "openai-compatible": ("openai", "openai", "openai"), + "openrouter": ("openai", "openai", "openai"), + "anthropic": ("anthropic", "anthropic", "anthropic"), + "bedrock": ("openai", "openai", "openai"), + "gemini": ("google.genai", "google-genai", "gemini"), +} + + @dataclass(frozen=True) class DiagnosticCheck: """One human- and machine-readable doctor result.""" @@ -152,6 +165,11 @@ def inspect_trace(path: Path | str) -> dict[str, Any]: return Trace.from_jsonl(path).summary() +def replay_trace(path: Path | str) -> dict[str, Any]: + """Reconstruct recorded turns without running tools, models, or network calls.""" + return Trace.from_jsonl(path).replay() + + def format_trace_summary(summary: dict[str, Any]) -> str: event_types = summary.get("event_types", {}) type_text = ", ".join(f"{name} x{count}" for name, count in event_types.items()) @@ -159,7 +177,9 @@ def format_trace_summary(summary: dict[str, Any]) -> str: "Chulk trace", f" path {summary.get('path')}", f" conversation {summary.get('conversation_id')}", + f" schemas {_format_schema_versions(summary.get('schema_versions'))}", f" events {summary.get('event_count')}", + f" sessions {summary.get('session_count')}", f" turns {summary.get('turn_count')}", f" failures {summary.get('failure_count')}", f" started {summary.get('started_at')}", @@ -172,6 +192,60 @@ def format_trace_summary(summary: dict[str, Any]) -> str: return "\n".join(lines) +def format_trace_replay(replay: dict[str, Any]) -> str: + """Format a deterministic, explicitly non-executing trace reconstruction.""" + lines = [ + "Chulk trace replay", + f" path {replay.get('path')}", + f" conversation {replay.get('conversation_id')}", + f" schemas {_format_schema_versions(replay.get('schema_versions'))}", + f" events {replay.get('event_count')}", + f" sessions {replay.get('session_count')}", + f" turns {replay.get('turn_count')}", + " mode read-only; no model, tool, or network execution", + ] + turns = replay.get("turns") + if isinstance(turns, list): + for index, turn in enumerate(turns, start=1): + if not isinstance(turn, dict): + continue + lines.append( + f" turn {index} {turn.get('turn_id')} [{turn.get('status', 'unknown')}]" + ) + if turn.get("user_message") is not None: + lines.extend( + [ + " user", + *[f" {line}" for line in str(turn["user_message"]).splitlines()], + ] + ) + lines.append(f" model calls {turn.get('model_request_count', 0)}") + tool_calls = turn.get("tool_calls") + if isinstance(tool_calls, list) and tool_calls: + lines.append(" tools") + for tool_call in tool_calls: + if not isinstance(tool_call, dict): + continue + lines.append( + f" {tool_call.get('tool_name') or 'unknown'} [{tool_call.get('status')}]" + ) + if turn.get("final_answer") is not None: + lines.extend( + [ + " answer", + *[f" {line}" for line in str(turn["final_answer"]).splitlines()], + ] + ) + lines.append(" warning replay output may contain sensitive runtime data") + return "\n".join(lines) + + +def _format_schema_versions(value: object) -> str: + if not isinstance(value, list) or not value: + return "unknown" + return ", ".join(str(item) for item in value) + + def export_trace_html( path: Path | str, *, @@ -195,25 +269,42 @@ def export_trace_html( def _provider_check(config: Config) -> DiagnosticCheck: providers = _configured_provider_models(config) - missing: list[str] = [] + missing_settings: list[str] = [] + missing_packages: list[str] = [] for label, provider, _model in providers: - if provider == "openai" and not config.openai_api_key: - missing.append(f"{label} openai (OPENAI_API_KEY)") - elif provider == "deepseek" and not config.deepseek_api_key: - missing.append(f"{label} deepseek (DEEPSEEK_API_KEY)") - if missing: + missing_settings.extend( + f"{label} {provider} ({setting})" + for setting in _missing_provider_settings(config, provider) + ) + if missing_package := _missing_provider_package(provider): + missing_packages.append(f"{label} {provider} ({missing_package})") + if missing_settings or missing_packages: + details = [] + remedies = [] + if missing_settings: + details.append( + "required provider settings are missing for: " + + ", ".join(missing_settings) + ) + remedies.append("Set the listed variables in the environment or project .env.") + if missing_packages: + details.append( + "required provider packages are missing for: " + + ", ".join(missing_packages) + ) + remedies.append("Install the listed provider extras.") return DiagnosticCheck( "provider", "fail", - "credentials are missing for: " + ", ".join(missing), - "Set the listed variables in the environment or project .env.", + "; ".join(details), + " ".join(remedies), ) if len(providers) == 1: provider = providers[0][1] detail = ( - f"{provider} credentials are set" - if provider in {"openai", "deepseek"} - else f"{provider} provider configured" + f"{provider} provider configured" + if provider == "local" + else f"{provider} provider requirements are set" ) return DiagnosticCheck("provider", "pass", detail) return DiagnosticCheck( @@ -223,6 +314,73 @@ def _provider_check(config: Config) -> DiagnosticCheck: ) +def _missing_provider_package(provider: str) -> str | None: + requirement = _PROVIDER_SDK_REQUIREMENTS.get(provider) + if requirement is None: + return None + module_name, package_name, extra_name = requirement + try: + available = find_spec(module_name) is not None + except (ImportError, ValueError): + available = False + if available: + return None + return f"{package_name}; install chulkharness[{extra_name}]" + + +def _missing_provider_settings(config: Config, provider: str) -> tuple[str, ...]: + """Return unresolved environment requirements for one configured provider.""" + if provider == "openai": + return () if _has_provider_value(config.openai_api_key) else ("OPENAI_API_KEY",) + if provider == "deepseek": + return ( + () + if _has_provider_value(config.deepseek_api_key) + else ("CHULK_DEEPSEEK_API_KEY or DEEPSEEK_API_KEY",) + ) + if provider == "local": + return () + if provider == "openai-compatible": + missing = [] + if not _has_provider_value(config.openai_compatible_api_key): + missing.append("CHULK_OPENAI_COMPATIBLE_API_KEY") + if not _has_provider_value(config.openai_compatible_base_url): + missing.append("CHULK_OPENAI_COMPATIBLE_BASE_URL") + return tuple(missing) + if provider == "openrouter": + return ( + () + if _has_provider_value(config.openrouter_api_key) + else ("CHULK_OPENROUTER_API_KEY or OPENROUTER_API_KEY",) + ) + if provider == "anthropic": + return ( + () + if _has_provider_value(config.anthropic_api_key) + else ("CHULK_ANTHROPIC_API_KEY or ANTHROPIC_API_KEY",) + ) + if provider == "bedrock": + missing = [] + if not _has_provider_value(config.bedrock_api_key): + missing.append( + "CHULK_BEDROCK_API_KEY, BEDROCK_API_KEY, or AWS_BEARER_TOKEN_BEDROCK" + ) + if not _has_provider_value(config.bedrock_base_url): + missing.append("CHULK_BEDROCK_BASE_URL or CHULK_BASE_URL") + return tuple(missing) + if provider == "gemini": + return ( + () + if _has_provider_value(config.gemini_api_key) + else ("CHULK_GEMINI_API_KEY, GEMINI_API_KEY, or GOOGLE_API_KEY",) + ) + return () + + +def _has_provider_value(value: str | None) -> bool: + return value is not None and bool(value.strip()) + + def _model_check(config: Config) -> DiagnosticCheck: providers = _configured_provider_models(config) capabilities = [] @@ -495,8 +653,10 @@ def _ensure_gitignore(path: Path) -> str: "export_trace_html", "format_doctor_report", "format_init_changes", + "format_trace_replay", "format_trace_summary", "initialize_project", "inspect_trace", + "replay_trace", "run_doctor", ] diff --git a/chulk/cli/parser.py b/chulk/cli/parser.py index 33f9983..e613c76 100644 --- a/chulk/cli/parser.py +++ b/chulk/cli/parser.py @@ -64,11 +64,17 @@ def _add_init_parser(subparsers: argparse._SubParsersAction) -> None: def _add_trace_parser(subparsers: argparse._SubParsersAction) -> None: - parser = subparsers.add_parser("trace", help="Inspect or export a JSONL trace.") + parser = subparsers.add_parser("trace", help="Inspect, replay, or export a JSONL trace.") trace_subparsers = parser.add_subparsers(dest="trace_command", required=True) inspect_parser = trace_subparsers.add_parser("inspect", help="Summarize a trace file.") inspect_parser.add_argument("path", help="Path to a Chulk JSONL trace.") inspect_parser.add_argument("--json", action="store_true", dest="json_output", help="Emit structured JSON.") + replay_parser = trace_subparsers.add_parser( + "replay", + help="Reconstruct recorded turns without executing them.", + ) + replay_parser.add_argument("path", help="Path to a Chulk JSONL trace.") + replay_parser.add_argument("--json", action="store_true", dest="json_output", help="Emit structured JSON.") export_parser = trace_subparsers.add_parser("export", help="Export a trace report.") export_parser.add_argument("path", help="Path to a Chulk JSONL trace.") export_parser.add_argument("--format", choices=("html",), default="html") diff --git a/chulk/cli/terminal.py b/chulk/cli/terminal.py index 126c82c..ad2aa01 100644 --- a/chulk/cli/terminal.py +++ b/chulk/cli/terminal.py @@ -19,6 +19,7 @@ from chulk.config import Config from chulk.core import Agent, TraceEvent +from chulk.llm.factory import provider_capabilities from chulk.sessions import ConversationRecord, MessageRecord from chulk.tools.permissions import PermissionDecisionRecord, PermissionRequest @@ -410,8 +411,8 @@ def _mcp_provider_path(config: Config) -> str: if not config.mcp_servers: return "none" providers = [config.llm_provider, *(provider.provider for provider in config.llm_fallback_providers)] - has_hosted = any(provider == "openai" for provider in providers) - has_bridge = any(provider != "openai" for provider in providers) + has_hosted = any(provider_capabilities(provider).supports_hosted_mcp_tools for provider in providers) + has_bridge = any(not provider_capabilities(provider).supports_hosted_mcp_tools for provider in providers) if has_hosted and has_bridge: return "hosted+bridge" return "hosted" if has_hosted else "bridge" diff --git a/chulk/config.py b/chulk/config.py index 95f11c1..0b6e3eb 100644 --- a/chulk/config.py +++ b/chulk/config.py @@ -8,6 +8,7 @@ from pathlib import Path from chulk.llm import supported_llm_providers +from chulk.llm.providers.compatible import DEFAULT_OPENROUTER_BASE_URL from chulk.mcp import MCPServerConfig, load_mcp_servers from chulk.tools.permissions import DEFAULT_PERMISSION_PROFILE, normalize_permission_profile @@ -63,6 +64,16 @@ class Config: deepseek_base_url: str = DEFAULT_DEEPSEEK_BASE_URL local_api_key: str | None = None local_base_url: str = DEFAULT_LOCAL_BASE_URL + openai_compatible_api_key: str | None = None + openai_compatible_base_url: str | None = None + openrouter_api_key: str | None = None + openrouter_base_url: str = DEFAULT_OPENROUTER_BASE_URL + anthropic_api_key: str | None = None + anthropic_base_url: str | None = None + bedrock_api_key: str | None = None + bedrock_base_url: str | None = None + gemini_api_key: str | None = None + gemini_base_url: str | None = None llm_fallback_providers: tuple[LLMFallbackProviderConfig, ...] = () history_limit: int = 20 max_tool_calls_per_turn: int = 5 @@ -153,8 +164,22 @@ def load_config(environ: Mapping[str, str] | None = None) -> Config: supported = ", ".join(sorted(SUPPORTED_LLM_PROVIDERS)) raise ConfigValueError("CHULK_LLM_PROVIDER", f"CHULK_LLM_PROVIDER must be one of: {supported}") - default_model = _default_model_for_provider(llm_provider) - model = env.get("CHULK_MODEL") or default_model + model = _configured_model(env, llm_provider) + bedrock_base_url = _bedrock_base_url(env) + llm_fallback_providers = _parse_fallback_providers( + env, + primary_provider=llm_provider, + primary_model=model, + ) + bedrock_is_configured = llm_provider == "bedrock" or any( + fallback.provider == "bedrock" for fallback in llm_fallback_providers + ) + if bedrock_is_configured and bedrock_base_url is None: + raise ConfigValueError( + "CHULK_BEDROCK_BASE_URL", + "CHULK_BEDROCK_BASE_URL or CHULK_BASE_URL is required when Bedrock " + "is configured as a primary or fallback provider", + ) runtime_dir = _resolve_config_path(env.get("CHULK_RUNTIME_DIR") or ".chulk", base=project_root) mcp_config_path = runtime_dir / "mcp.json" mcp_servers = load_mcp_servers(mcp_config_path, env) @@ -176,11 +201,27 @@ def load_config(environ: Mapping[str, str] | None = None) -> Config: deepseek_base_url=env.get("CHULK_DEEPSEEK_BASE_URL") or DEFAULT_DEEPSEEK_BASE_URL, local_api_key=env.get("CHULK_LOCAL_API_KEY") or None, local_base_url=env.get("CHULK_LOCAL_BASE_URL") or DEFAULT_LOCAL_BASE_URL, - llm_fallback_providers=_parse_fallback_providers( + openai_compatible_api_key=env.get("CHULK_OPENAI_COMPATIBLE_API_KEY") or None, + openai_compatible_base_url=env.get("CHULK_OPENAI_COMPATIBLE_BASE_URL") or None, + openrouter_api_key=env.get("CHULK_OPENROUTER_API_KEY") or env.get("OPENROUTER_API_KEY") or None, + openrouter_base_url=env.get("CHULK_OPENROUTER_BASE_URL") or DEFAULT_OPENROUTER_BASE_URL, + anthropic_api_key=env.get("CHULK_ANTHROPIC_API_KEY") or env.get("ANTHROPIC_API_KEY") or None, + anthropic_base_url=env.get("CHULK_ANTHROPIC_BASE_URL") or None, + bedrock_api_key=( + env.get("CHULK_BEDROCK_API_KEY") + or env.get("BEDROCK_API_KEY") + or env.get("AWS_BEARER_TOKEN_BEDROCK") + or None + ), + bedrock_base_url=bedrock_base_url, + gemini_api_key=_first_nonblank_env( env, - primary_provider=llm_provider, - primary_model=model, + "CHULK_GEMINI_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", ), + gemini_base_url=_first_nonblank_env(env, "CHULK_GEMINI_BASE_URL"), + llm_fallback_providers=llm_fallback_providers, history_limit=_env_int(env, "CHULK_HISTORY_LIMIT", 20), max_tool_calls_per_turn=_env_int(env, "CHULK_MAX_TOOL_CALLS_PER_TURN", 5), max_skills_per_turn=_env_int(env, "CHULK_MAX_SKILLS_PER_TURN", DEFAULT_MAX_SKILLS_PER_TURN), @@ -232,12 +273,40 @@ def _default_skills_dirs(project_skills_dir: Path) -> tuple[Path, ...]: return (bundled_skills_dir(), project_skills_dir) -def _default_model_for_provider(provider: str) -> str: - if provider == "deepseek": - return DEFAULT_DEEPSEEK_MODEL - if provider == "local": - return DEFAULT_LOCAL_MODEL - return DEFAULT_MODEL +def _configured_model(env: Mapping[str, str], provider: str) -> str: + configured = (env.get("CHULK_MODEL") or "").strip() + if configured: + return configured + default = _default_model_for_provider(provider) + if default is None: + raise ConfigValueError( + "CHULK_MODEL", + f"CHULK_MODEL is required when CHULK_LLM_PROVIDER={provider}", + ) + return default + + +def _default_model_for_provider(provider: str) -> str | None: + return { + "openai": DEFAULT_MODEL, + "deepseek": DEFAULT_DEEPSEEK_MODEL, + "local": DEFAULT_LOCAL_MODEL, + }.get(provider) + + +def _bedrock_base_url(env: Mapping[str, str]) -> str | None: + value = env.get("CHULK_BEDROCK_BASE_URL") or env.get("CHULK_BASE_URL") + if value is None or not value.strip(): + return None + return value.strip() + + +def _first_nonblank_env(env: Mapping[str, str], *keys: str) -> str | None: + for key in keys: + value = env.get(key) + if value is not None and value.strip(): + return value.strip() + return None def _parse_fallback_providers( @@ -258,12 +327,16 @@ def _parse_fallback_providers( continue provider, separator, raw_model = item.partition(":") provider = provider.strip().lower() - model = raw_model.strip() if separator else _default_model_for_provider(provider) if not provider: raise ValueError("CHULK_LLM_FALLBACK_PROVIDERS contains an empty provider name") if provider not in SUPPORTED_LLM_PROVIDERS: supported = ", ".join(sorted(SUPPORTED_LLM_PROVIDERS)) raise ValueError(f"CHULK_LLM_FALLBACK_PROVIDERS must use providers from: {supported}") + model = raw_model.strip() if separator else _default_model_for_provider(provider) + if model is None: + raise ValueError( + f"CHULK_LLM_FALLBACK_PROVIDERS entries for {provider} must include an explicit model" + ) if not model: raise ValueError("CHULK_LLM_FALLBACK_PROVIDERS entries with ':' must include a model") diff --git a/chulk/core/action_loop.py b/chulk/core/action_loop.py new file mode 100644 index 0000000..b93628a --- /dev/null +++ b/chulk/core/action_loop.py @@ -0,0 +1,470 @@ +"""Explicit model-action-tool execution loops for one agent turn.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from chulk.core.actions import AgentAction, FinalAnswerAction, PlanAction, PlanStepUpdateAction, ToolCallAction +from chulk.core.context import AgentPrompt +from chulk.core.events import TraceEvent +from chulk.core.planning import format_read_only_planning_tools +from chulk.core.state import ObservationRecord, PlanStep, ToolCallRecord, TurnState +from chulk.core.trace_format import format_action_trace, format_model_request_trace +from chulk.llm import LLMActionError, LLMActionResult +from chulk.tools.registry import ToolResult + +if TYPE_CHECKING: + from chulk.core.agent import Agent + + +def _record_model_request(agent: Agent, turn: TurnState, prompt: AgentPrompt) -> list[dict[str, str]]: + """Record the exact prompt and accounting metadata before one model request.""" + messages = prompt.messages + context_report = prompt.context_report.to_dict() + turn.context_reports.append(context_report) + agent.state.last_context_report = context_report + turn.model_request_count += 1 + agent._trace( + TraceEvent.MODEL_REQUEST_STARTED, + format_model_request_trace( + messages, + max_prompt_chars=agent.trace_max_prompt_chars, + request_index=turn.model_request_count, + turn_id=turn.turn_id, + loaded_memory_ids=agent.state.loaded_memory_ids, + loaded_skill_names=agent.state.loaded_skill_names, + available_tool_names=turn.available_tool_names, + context_report=context_report, + ), + ) + return messages + + +def _record_action_protocol_error(agent: Agent, turn: TurnState, exc: LLMActionError) -> str: + """Record one invalid model action and finish the turn consistently.""" + agent.state.json_repair_attempts += exc.repair_attempts + agent.state.errors.extend(f"JSON repair attempt: {error}" for error in exc.errors) + turn.errors.extend(f"JSON repair attempt: {error}" for error in exc.errors) + usage_payload, cost_payload = agent._record_model_accounting( + turn, + request_index=turn.model_request_count, + usage=exc.usage, + cost=exc.cost, + ) + if exc.raw_response: + agent._trace( + TraceEvent.MODEL_RESPONSE, + { + "turn_id": turn.turn_id, + "request_index": turn.model_request_count, + "content": exc.raw_response, + "repair_attempts": exc.repair_attempts, + "repair_errors": exc.errors, + "parse_failed": True, + "usage": usage_payload, + "cost": cost_payload, + }, + ) + return agent._fail_action_protocol_turn(exc, turn) + + +def _record_action_result(agent: Agent, turn: TurnState, action_result: LLMActionResult) -> AgentAction: + """Normalize action accounting and traces shared by sync and async loops.""" + action = action_result.action + agent.state.json_repair_attempts += action_result.repair_attempts + agent.state.errors.extend(f"JSON repair attempt: {error}" for error in action_result.errors) + turn.errors.extend(f"JSON repair attempt: {error}" for error in action_result.errors) + fallback_attempts = getattr(agent.llm_client, "last_attempts", None) + if fallback_attempts: + agent._trace( + TraceEvent.LLM_FALLBACK_ATTEMPTS, + { + "turn_id": turn.turn_id, + "request_index": turn.model_request_count, + "attempts": [ + attempt.to_dict() if hasattr(attempt, "to_dict") else {"attempt": str(attempt)} + for attempt in fallback_attempts + ], + }, + ) + usage_payload, cost_payload = agent._record_model_accounting( + turn, + request_index=turn.model_request_count, + usage=action_result.usage, + cost=action_result.cost, + fallback_attempts=fallback_attempts, + ) + agent._trace( + TraceEvent.MODEL_RESPONSE, + { + "turn_id": turn.turn_id, + "request_index": turn.model_request_count, + "content": action_result.raw_response, + "repair_attempts": action_result.repair_attempts, + "repair_errors": action_result.errors, + "usage": usage_payload, + "cost": cost_payload, + "metadata": action_result.metadata, + }, + ) + agent._trace(TraceEvent.PARSED_ACTION, format_action_trace(action)) + agent._trace(TraceEvent.MODEL_RESPONSE_PARSED, format_action_trace(action)) + return action + + +def _start_tool_call( + agent: Agent, + turn: TurnState, + action: ToolCallAction, + *, + phase: str, + require_plan: bool, +) -> tuple[ToolCallRecord, PlanStep | None]: + """Create and trace the durable record before a tool side effect starts.""" + turn.tool_call_count += 1 + plan_step = None if require_plan else agent._active_plan_step_for_tool(turn) + tool_call_record = ToolCallRecord( + tool_name=action.tool_name, + arguments=action.arguments, + iteration=turn.tool_call_count, + phase=phase, + plan_step_id=plan_step.id if plan_step else None, + ) + turn.tool_calls.append(tool_call_record) + agent._trace( + TraceEvent.TOOL_CALL_STARTED, + { + **tool_call_record.to_dict(), + "turn_id": turn.turn_id, + "max_tool_calls_per_turn": agent.max_tool_calls_per_turn, + }, + ) + return tool_call_record, plan_step + + +def _finish_tool_call( + agent: Agent, + turn: TurnState, + action: ToolCallAction, + *, + phase: str, + plan_step: PlanStep | None, + tool_call_record: ToolCallRecord, + result: ToolResult, +) -> str | None: + """Record a completed tool call, its observation, and plan evidence.""" + tool_call_record.finish(result) + state_tool_call: dict[str, object] = { + "tool_name": action.tool_name, + "arguments": action.arguments, + "phase": phase, + "success": result.success, + } + if tool_call_record.plan_step_id is not None: + state_tool_call["plan_step_id"] = tool_call_record.plan_step_id + agent.state.tool_calls.append(state_tool_call) + agent._trace( + TraceEvent.TOOL_CALL, + { + "turn_id": turn.turn_id, + "tool_name": action.tool_name, + "arguments": action.arguments, + "phase": phase, + "plan_step_id": tool_call_record.plan_step_id, + "success": result.success, + "error": result.error, + }, + ) + agent._trace( + TraceEvent.TOOL_CALL_COMPLETED if result.success else TraceEvent.TOOL_CALL_FAILED, + { + **tool_call_record.to_dict(), + "turn_id": turn.turn_id, + "max_tool_calls_per_turn": agent.max_tool_calls_per_turn, + }, + ) + observation, output_metadata = agent._format_tool_observation(action.tool_name, result) + agent.state.observations.append( + { + "tool_name": action.tool_name, + "observation": observation, + "output_metadata": output_metadata, + } + ) + turn.observations.append( + ObservationRecord( + tool_name=action.tool_name, + content=observation, + output_metadata=output_metadata, + ) + ) + agent.memory.add_observation(observation) + agent._trace( + TraceEvent.TOOL_OBSERVATION, + { + "turn_id": turn.turn_id, + "tool_name": action.tool_name, + "observation": observation, + "output_metadata": output_metadata, + }, + ) + if plan_step is None: + return None + if result.success: + agent._record_plan_tool_evidence(plan_step, tool_call_record, observation, output_metadata) + return None + return agent._handle_plan_step_tool_failure( + turn, + plan_step, + tool_call_record, + result, + observation, + output_metadata, + ) + + +def run_action_loop(agent: Agent, turn: TurnState, *, require_plan: bool) -> str: + """Run model/tool iterations until the turn pauses or completes.""" + while True: + blocked_response = agent._prepare_plan_execution_step(turn, require_plan=require_plan) + if blocked_response is not None: + return blocked_response + + prompt = agent._build_prompt(turn, require_plan=require_plan) + prompt = agent._compact_context_if_needed(prompt, turn, require_plan=require_plan) + messages = _record_model_request(agent, turn, prompt) + available_tools: list[object] = list(agent.tool_registry.list_tools()) + try: + action_result = agent.llm_client.complete_action( + messages, + max_repair_attempts=agent.max_json_repair_attempts, + tools=available_tools, + hosted_mcp_servers=agent.mcp_servers, + mcp_approval_callback=lambda approval_request: agent._resolve_hosted_mcp_approval( + approval_request, + turn, + ), + ) + except LLMActionError as exc: + return _record_action_protocol_error(agent, turn, exc) + action = _record_action_result(agent, turn, action_result) + + if isinstance(action, PlanAction): + return agent._handle_plan_action(action, turn, require_plan=require_plan) + + if isinstance(action, PlanStepUpdateAction): + step_update_response = agent._handle_plan_step_update(action, turn, require_plan=require_plan) + if step_update_response is not None: + return step_update_response + continue + + if isinstance(action, FinalAnswerAction): + if require_plan: + if turn.planning_feedback_count >= 2: + return agent._fail_turn("Planning failed because the model answered directly instead of returning a plan.", turn) + agent._request_plan_revision( + turn, + feedback=( + "Planning feedback: the user explicitly requested /plan, so do not answer directly. " + "Use read-only reconnaissance tools if codebase context is needed, then return a plan action " + "with concrete implementation steps that can be approved or rejected." + ), + ) + return agent._run_action_loop(turn, require_plan=True) + + if agent._approved_plan_incomplete(turn): + if turn.plan_execution_feedback_count >= 1: + return agent._fail_turn( + "Plan execution failed because the model returned a final answer before completing the approved plan.", + turn, + ) + agent._request_plan_execution_feedback( + turn, + feedback=( + "Plan execution feedback: the approved plan is not complete. " + "Continue the current executable step with a tool call, or return a plan_step_update " + "if the step's acceptance criteria are already satisfied. Do not return final_answer yet." + ), + ) + return agent._run_action_loop(turn, require_plan=False) + + if agent._final_answer_needs_revision(action.content, turn): + return agent._run_action_loop(turn, require_plan=False) + + return agent._complete_final_answer(action.content, turn) + + if isinstance(action, ToolCallAction): + if require_plan: + planning_tool_names = agent._read_only_planning_tool_names() + if action.tool_name not in planning_tool_names: + allowed_tools = format_read_only_planning_tools(planning_tool_names) + return agent._fail_turn( + "Planning can only use read-only reconnaissance tools before approval. " + f"Allowed planning tools: {allowed_tools}. " + "Return a plan action or retry with one of the allowed tools.", + turn, + ) + phase = "planning" + else: + phase = "execution" + + if agent._tool_call_count_for_phase(turn, phase) >= agent.max_tool_calls_per_turn: + if require_plan and phase == "planning" and not turn.planning_tool_limit_feedback_sent: + turn.planning_tool_limit_feedback_sent = True + agent._request_plan_revision( + turn, + feedback=( + "Planning feedback: the read-only reconnaissance tool budget is exhausted. " + "Do not call more tools. Return a plan action now using the context already gathered. " + "The plan must name concrete files/modules to change, behaviors to add, and tests to update." + ), + ) + return agent._run_action_loop(turn, require_plan=True) + return agent._fail_turn( + f"Tool call limit reached ({agent.max_tool_calls_per_turn}) " + f"during {phase} before a final answer.", + turn, + ) + tool_call_record, plan_step = _start_tool_call( + agent, + turn, + action, + phase=phase, + require_plan=require_plan, + ) + result = agent._execute_tool_with_retries(action.tool_name, action.arguments, turn) + blocked_response = _finish_tool_call( + agent, + turn, + action, + phase=phase, + plan_step=plan_step, + tool_call_record=tool_call_record, + result=result, + ) + if blocked_response is not None: + return blocked_response + + +async def run_action_loop_async(agent: Agent, turn: TurnState, *, require_plan: bool) -> str: + """Run model/tool iterations, awaiting async tool calls.""" + while True: + blocked_response = agent._prepare_plan_execution_step(turn, require_plan=require_plan) + if blocked_response is not None: + return blocked_response + + prompt = agent._build_prompt(turn, require_plan=require_plan) + prompt = await agent._compact_context_if_needed_async(prompt, turn, require_plan=require_plan) + messages = _record_model_request(agent, turn, prompt) + available_tools: list[object] = list(agent.tool_registry.list_tools()) + try: + action_result = await agent.llm_client.acomplete_action( + messages, + max_repair_attempts=agent.max_json_repair_attempts, + tools=available_tools, + hosted_mcp_servers=agent.mcp_servers, + mcp_approval_callback=lambda approval_request: agent._resolve_hosted_mcp_approval( + approval_request, + turn, + ), + ) + except LLMActionError as exc: + return _record_action_protocol_error(agent, turn, exc) + + action = _record_action_result(agent, turn, action_result) + + if isinstance(action, PlanAction): + return await agent._handle_plan_action_async(action, turn, require_plan=require_plan) + + if isinstance(action, PlanStepUpdateAction): + step_update_response = agent._handle_plan_step_update(action, turn, require_plan=require_plan) + if step_update_response is not None: + return step_update_response + continue + + if isinstance(action, FinalAnswerAction): + if require_plan: + if turn.planning_feedback_count >= 2: + return agent._fail_turn("Planning failed because the model answered directly instead of returning a plan.", turn) + agent._request_plan_revision( + turn, + feedback=( + "Planning feedback: the user explicitly requested /plan, so do not answer directly. " + "Use read-only reconnaissance tools if codebase context is needed, then return a plan action " + "with concrete implementation steps that can be approved or rejected." + ), + ) + return await agent._run_action_loop_async(turn, require_plan=True) + + if agent._approved_plan_incomplete(turn): + if turn.plan_execution_feedback_count >= 1: + return agent._fail_turn( + "Plan execution failed because the model returned a final answer before completing the approved plan.", + turn, + ) + agent._request_plan_execution_feedback( + turn, + feedback=( + "Plan execution feedback: the approved plan is not complete. " + "Continue the current executable step with a tool call, or return a plan_step_update " + "if the step's acceptance criteria are already satisfied. Do not return final_answer yet." + ), + ) + return await agent._run_action_loop_async(turn, require_plan=False) + + if await agent._final_answer_needs_revision_async(action.content, turn): + return await agent._run_action_loop_async(turn, require_plan=False) + + return agent._complete_final_answer(action.content, turn) + + if isinstance(action, ToolCallAction): + if require_plan: + planning_tool_names = agent._read_only_planning_tool_names() + if action.tool_name not in planning_tool_names: + allowed_tools = format_read_only_planning_tools(planning_tool_names) + return agent._fail_turn( + "Planning can only use read-only reconnaissance tools before approval. " + f"Allowed planning tools: {allowed_tools}. " + "Return a plan action or retry with one of the allowed tools.", + turn, + ) + phase = "planning" + else: + phase = "execution" + + if agent._tool_call_count_for_phase(turn, phase) >= agent.max_tool_calls_per_turn: + if require_plan and phase == "planning" and not turn.planning_tool_limit_feedback_sent: + turn.planning_tool_limit_feedback_sent = True + agent._request_plan_revision( + turn, + feedback=( + "Planning feedback: the read-only reconnaissance tool budget is exhausted. " + "Do not call more tools. Return a plan action now using the context already gathered. " + "The plan must name concrete files/modules to change, behaviors to add, and tests to update." + ), + ) + return await agent._run_action_loop_async(turn, require_plan=True) + return agent._fail_turn( + f"Tool call limit reached ({agent.max_tool_calls_per_turn}) " + f"during {phase} before a final answer.", + turn, + ) + tool_call_record, plan_step = _start_tool_call( + agent, + turn, + action, + phase=phase, + require_plan=require_plan, + ) + result = await agent._execute_tool_with_retries_async(action.tool_name, action.arguments, turn) + blocked_response = _finish_tool_call( + agent, + turn, + action, + phase=phase, + plan_step=plan_step, + tool_call_record=tool_call_record, + result=result, + ) + if blocked_response is not None: + return blocked_response diff --git a/chulk/core/actions.py b/chulk/core/actions.py index 5c635c7..e1bc44b 100644 --- a/chulk/core/actions.py +++ b/chulk/core/actions.py @@ -333,4 +333,4 @@ def _coerce_retry_limit(value: Any) -> int: raise ActionParseError("plan step retry_limit must be an integer") if value < 0: raise ActionParseError("plan step retry_limit cannot be negative") - return 0 + return value diff --git a/chulk/core/agent.py b/chulk/core/agent.py index 234c075..310bec9 100644 --- a/chulk/core/agent.py +++ b/chulk/core/agent.py @@ -11,12 +11,17 @@ from dataclasses import replace import re import time +from typing import Any, cast -from chulk.core.actions import FinalAnswerAction, PlanAction, PlanStepUpdateAction, ToolCallAction +from chulk.core.action_loop import run_action_loop, run_action_loop_async +from chulk.core.actions import PlanAction, PlanStepUpdateAction from chulk.core.context import AgentPrompt, ContextBudget, TurnContextSection from chulk.core.events import AgentEvent, TraceEvent from chulk.core.observations import format_tool_observation -from chulk.core.planning import READ_ONLY_PLANNING_TOOL_NAMES, format_read_only_planning_tools, plan_looks_like_reconnaissance +from chulk.core.planning import ( + plan_looks_like_reconnaissance, + read_only_planning_tool_names, +) from chulk.core.prompt_builder import build_agent_prompt from chulk.core.prompts import BASE_SYSTEM_PROMPT from chulk.core.reflection import ( @@ -26,7 +31,7 @@ parse_reflection_response, ) from chulk.core.state import AgentState, ObservationRecord, PlanStep, ToolCallRecord, TurnState, utc_now -from chulk.core.trace_format import format_action_trace, format_model_request_trace +from chulk.core.trace_format import format_model_request_trace from chulk.llm import LLMCost, LLMActionError, LLMClient, LLMError, LLMUsage from chulk.llm.usage import aggregate_cost, aggregate_usage, cost_from_dict, usage_from_dict from chulk.mcp import MCPServerConfig @@ -168,6 +173,11 @@ def close(self) -> None: close() except Exception as exc: # pragma: no cover - defensive aggregation failures.append(exc) + if self.trace_logger is not None: + try: + self.trace_logger.close() + except Exception as exc: # pragma: no cover - defensive aggregation + failures.append(exc) self.event_callback = None self.event_sink = None self._tool_contexts.clear() @@ -236,6 +246,14 @@ def run_planned_turn(self, user_message: str) -> str: raise ValueError("user_message cannot be empty") return self._run_user_turn(clean_message, require_plan=True) + async def run_planned_turn_async(self, user_message: str) -> str: + """Run one planned turn through async model and tool transports.""" + self._ensure_open() + clean_message = user_message.strip() + if not clean_message: + raise ValueError("user_message cannot be empty") + return await self._run_user_turn_async(clean_message, require_plan=True) + def _run_user_turn( self, clean_message: str, @@ -448,533 +466,11 @@ def describe_plan_status(self) -> str: def _run_action_loop(self, turn: TurnState, *, require_plan: bool) -> str: """Run model/tool iterations until the turn pauses or completes.""" - while True: - blocked_response = self._prepare_plan_execution_step(turn, require_plan=require_plan) - if blocked_response is not None: - return blocked_response - - prompt = self._build_prompt(turn, require_plan=require_plan) - prompt = self._compact_context_if_needed(prompt, turn, require_plan=require_plan) - messages = prompt.messages - context_report = prompt.context_report.to_dict() - turn.context_reports.append(context_report) - self.state.last_context_report = context_report - turn.model_request_count += 1 - self._trace( - TraceEvent.MODEL_REQUEST_STARTED, - format_model_request_trace( - messages, - max_prompt_chars=self.trace_max_prompt_chars, - request_index=turn.model_request_count, - turn_id=turn.turn_id, - loaded_memory_ids=self.state.loaded_memory_ids, - loaded_skill_names=self.state.loaded_skill_names, - available_tool_names=turn.available_tool_names, - context_report=context_report, - ), - ) - try: - action_result = self.llm_client.complete_action( - messages, - max_repair_attempts=self.max_json_repair_attempts, - tools=self.tool_registry.list_tools(), - hosted_mcp_servers=self.mcp_servers, - mcp_approval_callback=lambda approval_request: self._resolve_hosted_mcp_approval( - approval_request, - turn, - ), - ) - except LLMActionError as exc: - self.state.json_repair_attempts += exc.repair_attempts - self.state.errors.extend(f"JSON repair attempt: {error}" for error in exc.errors) - turn.errors.extend(f"JSON repair attempt: {error}" for error in exc.errors) - usage_payload, cost_payload = self._record_model_accounting( - turn, - request_index=turn.model_request_count, - usage=exc.usage, - cost=exc.cost, - ) - if exc.raw_response: - self._trace( - TraceEvent.MODEL_RESPONSE, - { - "turn_id": turn.turn_id, - "request_index": turn.model_request_count, - "content": exc.raw_response, - "repair_attempts": exc.repair_attempts, - "repair_errors": exc.errors, - "parse_failed": True, - "usage": usage_payload, - "cost": cost_payload, - }, - ) - return self._fail_action_protocol_turn(exc, turn) - action = action_result.action - self.state.json_repair_attempts += action_result.repair_attempts - self.state.errors.extend(f"JSON repair attempt: {error}" for error in action_result.errors) - turn.errors.extend(f"JSON repair attempt: {error}" for error in action_result.errors) - fallback_attempts = getattr(self.llm_client, "last_attempts", None) - if fallback_attempts: - self._trace( - TraceEvent.LLM_FALLBACK_ATTEMPTS, - { - "turn_id": turn.turn_id, - "request_index": turn.model_request_count, - "attempts": [ - attempt.to_dict() if hasattr(attempt, "to_dict") else {"attempt": str(attempt)} - for attempt in fallback_attempts - ], - }, - ) - usage_payload, cost_payload = self._record_model_accounting( - turn, - request_index=turn.model_request_count, - usage=action_result.usage, - cost=action_result.cost, - fallback_attempts=fallback_attempts, - ) - self._trace( - TraceEvent.MODEL_RESPONSE, - { - "turn_id": turn.turn_id, - "request_index": turn.model_request_count, - "content": action_result.raw_response, - "repair_attempts": action_result.repair_attempts, - "repair_errors": action_result.errors, - "usage": usage_payload, - "cost": cost_payload, - "metadata": action_result.metadata, - }, - ) - self._trace(TraceEvent.PARSED_ACTION, format_action_trace(action)) - self._trace(TraceEvent.MODEL_RESPONSE_PARSED, format_action_trace(action)) - - if isinstance(action, PlanAction): - return self._handle_plan_action(action, turn, require_plan=require_plan) - - if isinstance(action, PlanStepUpdateAction): - step_update_response = self._handle_plan_step_update(action, turn, require_plan=require_plan) - if step_update_response is not None: - return step_update_response - continue - - if isinstance(action, FinalAnswerAction): - if require_plan: - if turn.planning_feedback_count >= 2: - return self._fail_turn("Planning failed because the model answered directly instead of returning a plan.", turn) - self._request_plan_revision( - turn, - feedback=( - "Planning feedback: the user explicitly requested /plan, so do not answer directly. " - "Use read-only reconnaissance tools if codebase context is needed, then return a plan action " - "with concrete implementation steps that can be approved or rejected." - ), - ) - return self._run_action_loop(turn, require_plan=True) - - if self._approved_plan_incomplete(turn): - if turn.plan_execution_feedback_count >= 1: - return self._fail_turn( - "Plan execution failed because the model returned a final answer before completing the approved plan.", - turn, - ) - self._request_plan_execution_feedback( - turn, - feedback=( - "Plan execution feedback: the approved plan is not complete. " - "Continue the current executable step with a tool call, or return a plan_step_update " - "if the step's acceptance criteria are already satisfied. Do not return final_answer yet." - ), - ) - return self._run_action_loop(turn, require_plan=False) - - if self._final_answer_needs_revision(action.content, turn): - return self._run_action_loop(turn, require_plan=False) - - return self._complete_final_answer(action.content, turn) - - if isinstance(action, ToolCallAction): - if require_plan: - if action.tool_name not in READ_ONLY_PLANNING_TOOL_NAMES: - allowed_tools = format_read_only_planning_tools() - return self._fail_turn( - "Planning can only use read-only reconnaissance tools before approval. " - f"Allowed planning tools: {allowed_tools}. " - "Return a plan action or retry with one of the allowed tools.", - turn, - ) - phase = "planning" - else: - phase = "execution" - - if self._tool_call_count_for_phase(turn, phase) >= self.max_tool_calls_per_turn: - if require_plan and phase == "planning" and not turn.planning_tool_limit_feedback_sent: - turn.planning_tool_limit_feedback_sent = True - self._request_plan_revision( - turn, - feedback=( - "Planning feedback: the read-only reconnaissance tool budget is exhausted. " - "Do not call more tools. Return a plan action now using the context already gathered. " - "The plan must name concrete files/modules to change, behaviors to add, and tests to update." - ), - ) - return self._run_action_loop(turn, require_plan=True) - return self._fail_turn( - f"Tool call limit reached ({self.max_tool_calls_per_turn}) " - f"during {phase} before a final answer.", - turn, - ) - turn.tool_call_count += 1 - plan_step = None if require_plan else self._active_plan_step_for_tool(turn) - tool_call_record = ToolCallRecord( - tool_name=action.tool_name, - arguments=action.arguments, - iteration=turn.tool_call_count, - phase=phase, - plan_step_id=plan_step.id if plan_step else None, - ) - turn.tool_calls.append(tool_call_record) - tool_call_payload = { - **tool_call_record.to_dict(), - "turn_id": turn.turn_id, - "max_tool_calls_per_turn": self.max_tool_calls_per_turn, - } - self._trace(TraceEvent.TOOL_CALL_STARTED, tool_call_payload) - result = self._execute_tool_with_retries(action.tool_name, action.arguments, turn) - tool_call_record.finish(result) - state_tool_call = { - "tool_name": action.tool_name, - "arguments": action.arguments, - "phase": phase, - "success": result.success, - } - if tool_call_record.plan_step_id is not None: - state_tool_call["plan_step_id"] = tool_call_record.plan_step_id - self.state.tool_calls.append(state_tool_call) - self._trace( - TraceEvent.TOOL_CALL, - { - "turn_id": turn.turn_id, - "tool_name": action.tool_name, - "arguments": action.arguments, - "phase": phase, - "plan_step_id": tool_call_record.plan_step_id, - "success": result.success, - "error": result.error, - }, - ) - completion_payload = { - **tool_call_record.to_dict(), - "turn_id": turn.turn_id, - "max_tool_calls_per_turn": self.max_tool_calls_per_turn, - } - self._trace( - TraceEvent.TOOL_CALL_COMPLETED if result.success else TraceEvent.TOOL_CALL_FAILED, - completion_payload, - ) - observation, output_metadata = self._format_tool_observation(action.tool_name, result) - self.state.observations.append( - { - "tool_name": action.tool_name, - "observation": observation, - "output_metadata": output_metadata, - } - ) - observation_record = ObservationRecord( - tool_name=action.tool_name, - content=observation, - output_metadata=output_metadata, - ) - turn.observations.append(observation_record) - self.memory.add_observation(observation) - self._trace( - TraceEvent.TOOL_OBSERVATION, - { - "turn_id": turn.turn_id, - "tool_name": action.tool_name, - "observation": observation, - "output_metadata": output_metadata, - }, - ) - if plan_step is not None: - if result.success: - self._record_plan_tool_evidence(plan_step, tool_call_record, observation, output_metadata) - else: - blocked_response = self._block_plan_step_after_tool_failure( - turn, - plan_step, - tool_call_record, - result, - observation, - output_metadata, - ) - return blocked_response + return run_action_loop(self, turn, require_plan=require_plan) async def _run_action_loop_async(self, turn: TurnState, *, require_plan: bool) -> str: """Run model/tool iterations, awaiting async tool calls.""" - while True: - blocked_response = self._prepare_plan_execution_step(turn, require_plan=require_plan) - if blocked_response is not None: - return blocked_response - - prompt = self._build_prompt(turn, require_plan=require_plan) - prompt = await self._compact_context_if_needed_async(prompt, turn, require_plan=require_plan) - messages = prompt.messages - context_report = prompt.context_report.to_dict() - turn.context_reports.append(context_report) - self.state.last_context_report = context_report - turn.model_request_count += 1 - self._trace( - TraceEvent.MODEL_REQUEST_STARTED, - format_model_request_trace( - messages, - max_prompt_chars=self.trace_max_prompt_chars, - request_index=turn.model_request_count, - turn_id=turn.turn_id, - loaded_memory_ids=self.state.loaded_memory_ids, - loaded_skill_names=self.state.loaded_skill_names, - available_tool_names=turn.available_tool_names, - context_report=context_report, - ), - ) - try: - action_result = await asyncio.to_thread( - self.llm_client.complete_action, - messages, - max_repair_attempts=self.max_json_repair_attempts, - tools=self.tool_registry.list_tools(), - hosted_mcp_servers=self.mcp_servers, - mcp_approval_callback=lambda approval_request: self._resolve_hosted_mcp_approval( - approval_request, - turn, - ), - ) - except LLMActionError as exc: - self.state.json_repair_attempts += exc.repair_attempts - self.state.errors.extend(f"JSON repair attempt: {error}" for error in exc.errors) - turn.errors.extend(f"JSON repair attempt: {error}" for error in exc.errors) - usage_payload, cost_payload = self._record_model_accounting( - turn, - request_index=turn.model_request_count, - usage=exc.usage, - cost=exc.cost, - ) - if exc.raw_response: - self._trace( - TraceEvent.MODEL_RESPONSE, - { - "turn_id": turn.turn_id, - "request_index": turn.model_request_count, - "content": exc.raw_response, - "repair_attempts": exc.repair_attempts, - "repair_errors": exc.errors, - "parse_failed": True, - "usage": usage_payload, - "cost": cost_payload, - }, - ) - return self._fail_action_protocol_turn(exc, turn) - - action = action_result.action - self.state.json_repair_attempts += action_result.repair_attempts - self.state.errors.extend(f"JSON repair attempt: {error}" for error in action_result.errors) - turn.errors.extend(f"JSON repair attempt: {error}" for error in action_result.errors) - fallback_attempts = getattr(self.llm_client, "last_attempts", None) - if fallback_attempts: - self._trace( - TraceEvent.LLM_FALLBACK_ATTEMPTS, - { - "turn_id": turn.turn_id, - "request_index": turn.model_request_count, - "attempts": [ - attempt.to_dict() if hasattr(attempt, "to_dict") else {"attempt": str(attempt)} - for attempt in fallback_attempts - ], - }, - ) - usage_payload, cost_payload = self._record_model_accounting( - turn, - request_index=turn.model_request_count, - usage=action_result.usage, - cost=action_result.cost, - fallback_attempts=fallback_attempts, - ) - self._trace( - TraceEvent.MODEL_RESPONSE, - { - "turn_id": turn.turn_id, - "request_index": turn.model_request_count, - "content": action_result.raw_response, - "repair_attempts": action_result.repair_attempts, - "repair_errors": action_result.errors, - "usage": usage_payload, - "cost": cost_payload, - "metadata": action_result.metadata, - }, - ) - self._trace(TraceEvent.PARSED_ACTION, format_action_trace(action)) - self._trace(TraceEvent.MODEL_RESPONSE_PARSED, format_action_trace(action)) - - if isinstance(action, PlanAction): - return self._handle_plan_action(action, turn, require_plan=require_plan) - - if isinstance(action, PlanStepUpdateAction): - step_update_response = self._handle_plan_step_update(action, turn, require_plan=require_plan) - if step_update_response is not None: - return step_update_response - continue - - if isinstance(action, FinalAnswerAction): - if require_plan: - if turn.planning_feedback_count >= 2: - return self._fail_turn("Planning failed because the model answered directly instead of returning a plan.", turn) - self._request_plan_revision( - turn, - feedback=( - "Planning feedback: the user explicitly requested /plan, so do not answer directly. " - "Use read-only reconnaissance tools if codebase context is needed, then return a plan action " - "with concrete implementation steps that can be approved or rejected." - ), - ) - return await self._run_action_loop_async(turn, require_plan=True) - - if self._approved_plan_incomplete(turn): - if turn.plan_execution_feedback_count >= 1: - return self._fail_turn( - "Plan execution failed because the model returned a final answer before completing the approved plan.", - turn, - ) - self._request_plan_execution_feedback( - turn, - feedback=( - "Plan execution feedback: the approved plan is not complete. " - "Continue the current executable step with a tool call, or return a plan_step_update " - "if the step's acceptance criteria are already satisfied. Do not return final_answer yet." - ), - ) - return await self._run_action_loop_async(turn, require_plan=False) - - if await self._final_answer_needs_revision_async(action.content, turn): - return await self._run_action_loop_async(turn, require_plan=False) - - return self._complete_final_answer(action.content, turn) - - if isinstance(action, ToolCallAction): - if require_plan: - if action.tool_name not in READ_ONLY_PLANNING_TOOL_NAMES: - allowed_tools = format_read_only_planning_tools() - return self._fail_turn( - "Planning can only use read-only reconnaissance tools before approval. " - f"Allowed planning tools: {allowed_tools}. " - "Return a plan action or retry with one of the allowed tools.", - turn, - ) - phase = "planning" - else: - phase = "execution" - - if self._tool_call_count_for_phase(turn, phase) >= self.max_tool_calls_per_turn: - if require_plan and phase == "planning" and not turn.planning_tool_limit_feedback_sent: - turn.planning_tool_limit_feedback_sent = True - self._request_plan_revision( - turn, - feedback=( - "Planning feedback: the read-only reconnaissance tool budget is exhausted. " - "Do not call more tools. Return a plan action now using the context already gathered. " - "The plan must name concrete files/modules to change, behaviors to add, and tests to update." - ), - ) - return await self._run_action_loop_async(turn, require_plan=True) - return self._fail_turn( - f"Tool call limit reached ({self.max_tool_calls_per_turn}) " - f"during {phase} before a final answer.", - turn, - ) - turn.tool_call_count += 1 - plan_step = None if require_plan else self._active_plan_step_for_tool(turn) - tool_call_record = ToolCallRecord( - tool_name=action.tool_name, - arguments=action.arguments, - iteration=turn.tool_call_count, - phase=phase, - plan_step_id=plan_step.id if plan_step else None, - ) - turn.tool_calls.append(tool_call_record) - tool_call_payload = { - **tool_call_record.to_dict(), - "turn_id": turn.turn_id, - "max_tool_calls_per_turn": self.max_tool_calls_per_turn, - } - self._trace(TraceEvent.TOOL_CALL_STARTED, tool_call_payload) - result = await self._execute_tool_with_retries_async(action.tool_name, action.arguments, turn) - tool_call_record.finish(result) - state_tool_call = { - "tool_name": action.tool_name, - "arguments": action.arguments, - "phase": phase, - "success": result.success, - } - if tool_call_record.plan_step_id is not None: - state_tool_call["plan_step_id"] = tool_call_record.plan_step_id - self.state.tool_calls.append(state_tool_call) - self._trace( - TraceEvent.TOOL_CALL, - { - "turn_id": turn.turn_id, - "tool_name": action.tool_name, - "arguments": action.arguments, - "phase": phase, - "plan_step_id": tool_call_record.plan_step_id, - "success": result.success, - "error": result.error, - }, - ) - completion_payload = { - **tool_call_record.to_dict(), - "turn_id": turn.turn_id, - "max_tool_calls_per_turn": self.max_tool_calls_per_turn, - } - self._trace( - TraceEvent.TOOL_CALL_COMPLETED if result.success else TraceEvent.TOOL_CALL_FAILED, - completion_payload, - ) - observation, output_metadata = self._format_tool_observation(action.tool_name, result) - self.state.observations.append( - { - "tool_name": action.tool_name, - "observation": observation, - "output_metadata": output_metadata, - } - ) - observation_record = ObservationRecord( - tool_name=action.tool_name, - content=observation, - output_metadata=output_metadata, - ) - turn.observations.append(observation_record) - self.memory.add_observation(observation) - self._trace( - TraceEvent.TOOL_OBSERVATION, - { - "turn_id": turn.turn_id, - "tool_name": action.tool_name, - "observation": observation, - "output_metadata": output_metadata, - }, - ) - if plan_step is not None: - if result.success: - self._record_plan_tool_evidence(plan_step, tool_call_record, observation, output_metadata) - else: - blocked_response = self._block_plan_step_after_tool_failure( - turn, - plan_step, - tool_call_record, - result, - observation, - output_metadata, - ) - return blocked_response + return await run_action_loop_async(self, turn, require_plan=require_plan) def _build_messages(self, turn: TurnState, *, require_plan: bool) -> list[dict[str, str]]: """Build the model input from prompt, tools, and short-term history.""" @@ -1195,7 +691,7 @@ async def _summarize_context_messages_async( request_payload["summary_source_message_count"] = len(messages) self._trace(TraceEvent.MODEL_REQUEST_STARTED, request_payload) try: - response = await asyncio.to_thread(self.llm_client.complete_response, summary_messages) + response = await self.llm_client.acomplete_response(summary_messages) raw_summary = response.content except LLMError as exc: self._trace( @@ -1255,6 +751,33 @@ def _handle_plan_action(self, action: PlanAction, turn: TurnState, *, require_pl self._trace(TraceEvent.PLAN_CREATED, {"turn_id": turn.turn_id, "plan": plan.to_dict(), "turn": turn.to_dict()}) return response + async def _handle_plan_action_async( + self, + action: PlanAction, + turn: TurnState, + *, + require_plan: bool, + ) -> str: + """Handle a proposed plan without falling back to the sync action loop.""" + if not require_plan: + return self._fail_turn("Model proposed a new plan after execution had already been approved.", turn) + + plan = action.plan + if self._plan_needs_revision(plan, turn): + if turn.planning_feedback_count >= 2: + return self._fail_turn("Planning failed because the model kept proposing reconnaissance as the plan.", turn) + self._request_plan_revision(turn, plan=plan) + return await self._run_action_loop_async(turn, require_plan=True) + + turn.wait_for_plan_approval(plan) + self.state.active_plan = plan + self.state.pending_plan_turn_id = turn.turn_id + response = plan.to_user_text() + "\n\nUse /approve to execute this plan or /reject to cancel it." + self.memory.add_assistant_message(response) + self.state.messages = self.memory.recent() + self._trace(TraceEvent.PLAN_CREATED, {"turn_id": turn.turn_id, "plan": plan.to_dict(), "turn": turn.to_dict()}) + return response + def _permission_result_for_tool_call( self, tool_name: str, @@ -1673,7 +1196,7 @@ async def _reflect_before_final_answer_async(self, proposed_answer: str, turn: T self._trace(TraceEvent.MODEL_REQUEST_STARTED, request_payload) try: - response = await asyncio.to_thread(self.llm_client.complete_response, messages) + response = await self.llm_client.acomplete_response(messages) raw_response = response.content except LLMError as exc: return self._fail_open_reflection( @@ -1882,6 +1405,9 @@ def _clear_active_plan_for_turn(self, turn: TurnState) -> None: def _tool_call_count_for_phase(self, turn: TurnState, phase: str) -> int: return sum(1 for tool_call in turn.tool_calls if tool_call.phase == phase) + def _read_only_planning_tool_names(self) -> frozenset[str]: + return read_only_planning_tool_names(self.tool_registry.list_tools()) + def _prepare_plan_execution_step(self, turn: TurnState, *, require_plan: bool) -> str | None: if require_plan: return None @@ -1926,16 +1452,21 @@ def _record_plan_tool_evidence( tool_call_record: ToolCallRecord, observation: str, output_metadata: dict, + *, + retry_metadata: dict | None = None, ) -> None: + metadata = { + "phase": tool_call_record.phase, + "output_metadata": output_metadata, + "tool_call": tool_call_record.to_dict(), + } + if retry_metadata is not None: + metadata["plan_step_retry"] = retry_metadata step.add_evidence( observation, tool_name=tool_call_record.tool_name, tool_call_iteration=tool_call_record.iteration, - metadata={ - "phase": tool_call_record.phase, - "output_metadata": output_metadata, - "tool_call": tool_call_record.to_dict(), - }, + metadata=metadata, ) def _handle_plan_step_update( @@ -1966,7 +1497,7 @@ def _handle_plan_step_update( f"Current step id is {step.id}; the model tried to update {action.step_id}." ), ) - return + return None step.add_evidence(action.evidence, tool_name="plan_step_update") if action.status == "completed": @@ -1978,7 +1509,7 @@ def _handle_plan_step_update( self._trace_plan_step_event(turn, step, TraceEvent.PLAN_STEP_BLOCKED) return self._block_turn(_format_plan_step_blocked_message(step), turn) - def _block_plan_step_after_tool_failure( + def _handle_plan_step_tool_failure( self, turn: TurnState, step: PlanStep, @@ -1986,10 +1517,69 @@ def _block_plan_step_after_tool_failure( result: ToolResult, observation: str, output_metadata: dict, - ) -> str: - self._record_plan_tool_evidence(step, tool_call_record, observation, output_metadata) + ) -> str | None: reason = _format_tool_failure_reason(result) - step.block(reason) + retries_used = step.retry_count + retry_number = retries_used + 1 + tool_calls_remaining = max( + 0, + self.max_tool_calls_per_turn - self._tool_call_count_for_phase(turn, tool_call_record.phase), + ) + retry_scheduled = retry_number <= step.retry_limit and tool_calls_remaining > 0 + if retry_scheduled: + disposition = "retry_scheduled" + retries_used = retry_number + elif retry_number > step.retry_limit: + disposition = "retry_limit_exhausted" + else: + disposition = "tool_call_limit_exhausted" + retry_metadata = { + "disposition": disposition, + "failure_number": step.tool_failure_count + 1, + "retry_limit": step.retry_limit, + "retries_used": retries_used, + "retries_remaining": max(0, step.retry_limit - retries_used), + "tool_calls_remaining": tool_calls_remaining, + "failure_kind": result.failure_kind, + "error": result.error, + } + self._record_plan_tool_evidence( + step, + tool_call_record, + observation, + output_metadata, + retry_metadata=retry_metadata, + ) + + if retry_scheduled: + self._add_synthetic_observation( + turn, + tool_name="plan_step_retry", + content=( + f"Plan step {step.id} remains in_progress after {reason} " + f"Recovery attempt {retries_used} of {step.retry_limit} is available; " + f"{step.retries_remaining} retries remain. Correct the arguments or choose a safe " + "alternative that can satisfy the current step's acceptance criteria." + ), + output_metadata={ + "step_id": step.id, + "step_status": step.status, + **retry_metadata, + }, + ) + return None + + blocked_reason = reason + if step.retry_limit: + if disposition == "retry_limit_exhausted": + retry_label = "retry" if step.retry_count == 1 else "retries" + blocked_reason = f"{reason} Step retry limit exhausted after {step.retry_count} {retry_label}." + else: + blocked_reason = ( + f"{reason} No step retry could run because the execution tool-call limit " + f"({self.max_tool_calls_per_turn}) was reached." + ) + step.block(blocked_reason) self._trace_plan_step_event( turn, step, @@ -2343,7 +1933,9 @@ def _aggregate_model_usage_reports(reports: list[dict]) -> dict: } -def _coerce_turn_context_sections(values: list[TurnContextSection | dict | str] | None) -> list[TurnContextSection]: +def _coerce_turn_context_sections( + values: list[TurnContextSection | dict[str, Any] | str] | None, +) -> list[TurnContextSection]: if not values: return [] sections: list[TurnContextSection] = [] @@ -2359,7 +1951,8 @@ def _coerce_turn_context_sections(values: list[TurnContextSection | dict | str] if not isinstance(content, str) or not content.strip(): continue section_id = value.get("id") or value.get("source_id") or f"context-{index}" - metadata = value.get("metadata") if isinstance(value.get("metadata"), dict) else {} + raw_metadata = value.get("metadata") + metadata = cast(dict[str, Any], raw_metadata) if isinstance(raw_metadata, dict) else {} sections.append( TurnContextSection( id=str(section_id), diff --git a/chulk/core/events.py b/chulk/core/events.py index 706bc0d..3013f30 100644 --- a/chulk/core/events.py +++ b/chulk/core/events.py @@ -17,6 +17,8 @@ class AgentEvent: class TraceEvent: """Internal trace/progress constants; not a public compatibility catalog.""" + SESSION_STARTED = "session_started" + SESSION_FINISHED = "session_finished" TURN_STARTED = "turn_started" USER_MESSAGE = "user_message" TURN_CONTEXT_SELECTED = "turn_context_selected" diff --git a/chulk/core/observations.py b/chulk/core/observations.py index aafb9c3..326d28d 100644 --- a/chulk/core/observations.py +++ b/chulk/core/observations.py @@ -20,11 +20,11 @@ def format_tool_observation( max_stdout_chars: int, max_stderr_chars: int, artifact_writer: ArtifactWriter, -) -> tuple[str, dict]: +) -> tuple[str, dict[str, Any]]: """Format one tool result as a bounded model observation plus metadata.""" status = "success" if result.success else "error" parts = [f"Tool {result.tool_name} finished with {status}.", result.observation] - metadata = { + metadata: dict[str, Any] = { "requested_tool_name": requested_tool_name, "tool_name": result.tool_name, "success": result.success, @@ -74,7 +74,7 @@ def format_tool_observation( def _append_artifact_note( parts: list[str], - metadata: dict, + metadata: dict[str, Any], artifact_writer: ArtifactWriter, tool_name: str, field: str, @@ -91,7 +91,7 @@ def _append_artifact_note( parts.append(_artifact_note(field, artifact)) -def _artifact_note(field: str, artifact: dict) -> str: +def _artifact_note(field: str, artifact: dict[str, Any]) -> str: return ( f"[full {field} saved to {artifact['path']}; " f"chars={artifact['char_count']}; sha256={artifact['sha256']}]" diff --git a/chulk/core/planning.py b/chulk/core/planning.py index 2faeea0..f62a0cb 100644 --- a/chulk/core/planning.py +++ b/chulk/core/planning.py @@ -2,20 +2,11 @@ from __future__ import annotations +from collections.abc import Iterable -READ_ONLY_PLANNING_TOOL_NAMES = frozenset( - { - "calculator", - "list_files", - "read_file", - "search_files", - "search_memory", - "list_memories", - "summarize_memories", - } -) +from chulk.tools.permissions import ToolPermissionLevel +from chulk.tools.registry import Tool -SUBSTANTIVE_RECONNAISSANCE_TOOL_NAMES = frozenset({"read_file", "search_files", "search_memory", "summarize_memories"}) RECONNAISSANCE_STEP_TERMS = frozenset( { @@ -56,9 +47,19 @@ ) -def format_read_only_planning_tools() -> str: - """Return read-only planning tool names for prompt text.""" - return ", ".join(sorted(READ_ONLY_PLANNING_TOOL_NAMES)) +def read_only_planning_tool_names(tools: Iterable[Tool]) -> frozenset[str]: + """Return registered tools whose declared permission metadata is read-only.""" + return frozenset( + tool.name + for tool in tools + if tool.normalized_permission_level() is ToolPermissionLevel.READ + ) + + +def format_read_only_planning_tools(tool_names: Iterable[str]) -> str: + """Return registered read-only planning tool names for prompt text.""" + names = sorted(set(tool_names)) + return ", ".join(names) if names else "none" def plan_step_looks_like_reconnaissance(title: str, description: str) -> bool: diff --git a/chulk/core/prompt_builder.py b/chulk/core/prompt_builder.py index 24cfff8..dbaa677 100644 --- a/chulk/core/prompt_builder.py +++ b/chulk/core/prompt_builder.py @@ -25,6 +25,7 @@ select_messages_for_budget, ) from chulk.core.state import Plan +from chulk.core.planning import read_only_planning_tool_names from chulk.memory import ConversationMemory, MemoryRecord from chulk.skills import Skill, SkillSelection from chulk.tools import ToolRegistry @@ -118,6 +119,7 @@ def build_agent_prompt( plan_approved=plan_approved, require_plan=require_plan, max_reconnaissance_tool_calls=max_tool_calls_per_turn, + read_only_tool_names=read_only_planning_tool_names(tool_registry.list_tools()), ) tool_rules = format_tool_call_rules(max_tool_calls_per_turn) tools_prompt = format_tools_for_prompt(tool_descriptions) diff --git a/chulk/core/prompts.py b/chulk/core/prompts.py index ab40d00..c19f11e 100644 --- a/chulk/core/prompts.py +++ b/chulk/core/prompts.py @@ -1,5 +1,6 @@ """Prompt templates for the agent loop.""" +from collections.abc import Iterable from html import escape import json @@ -56,7 +57,8 @@ For tool calls, use only argument fields from the listed tool schema. Use a plan only when the Planning section explicitly tells you to propose a plan. For plans, plan_json must be a JSON-encoded object string with summary and executable steps. -Each plan step should include depends_on, acceptance_criteria, and retry_limit 0. +Each plan step should include depends_on, acceptance_criteria, and a non-negative retry_limit. +Use retry_limit 0 for fail-fast steps. Otherwise keep it small; it counts model recovery attempts after a failed tool call. When an approved plan is active, work only on the current executable step. After tool evidence satisfies that step's acceptance criteria, return plan_step_update instead of final_answer. Use final_answer only after the approved plan is completed. @@ -75,7 +77,8 @@ When you need a listed Chulk tool, call that tool through the native tool interface. For tool calls, use only argument fields from the listed tool schema. When the Planning section tells you to propose a plan, call chulk_propose_plan. -For plans, provide a summary and executable steps with depends_on, acceptance_criteria, and retry_limit 0. +For plans, provide a summary and executable steps with depends_on, acceptance_criteria, and a non-negative retry_limit. +Use retry_limit 0 for fail-fast steps. Otherwise keep it small; it counts model recovery attempts after a failed tool call. When an approved plan is active, work only on the current executable step. After tool evidence satisfies that step's acceptance criteria, call chulk_plan_step_update instead of answering directly. Use a final answer only after the approved plan is completed. @@ -293,13 +296,14 @@ def format_planning_for_prompt( plan_approved: bool, require_plan: bool, max_reconnaissance_tool_calls: int, + read_only_tool_names: Iterable[str] = (), ) -> str: """Format one-shot planning instructions for prompt injection.""" if not planning_enabled: return "\n".join(["", "Planning: not requested for this turn.", ""]) if require_plan and active_plan is None: - read_only_tools = format_read_only_planning_tools() + read_only_tools = format_read_only_planning_tools(read_only_tool_names) return "\n".join( [ "", @@ -315,7 +319,8 @@ def format_planning_for_prompt( "Do not call shell, write, memory-mutation, import/export, or other mutating tools before approval.", "After reconnaissance, return a plan action. Do not execute implementation steps until the user approves the plan.", "Keep the plan concrete, short, and executable, with specific files/modules when they are known.", - "For each plan step, include depends_on, acceptance_criteria, and retry_limit 0.", + "For each plan step, include depends_on, acceptance_criteria, and a non-negative retry_limit.", + "Use retry_limit 0 for fail-fast steps. Otherwise keep it small; it counts recovery attempts after a failed tool call.", "Use dependencies only when a step truly cannot start until an earlier step is completed.", "The approval plan must be an implementation plan. Do not make read/list/search/explore/inspect steps the plan.", "Because the user explicitly requested /plan, do not answer directly. Return a plan action after any needed reconnaissance.", @@ -330,6 +335,7 @@ def format_planning_for_prompt( "Planning: approved for this turn.", "Follow the approved plan while executing this turn.", "Only work on the current executable step. Use tools until its acceptance criteria are satisfied.", + "If a tool fails and the step remains in_progress, inspect its retry budget and correct the next action. Do not claim the failed action succeeded.", "When the current step is satisfied, return a plan_step_update action for that step.", "Do not return a final_answer until every plan step is completed.", "", diff --git a/chulk/core/state.py b/chulk/core/state.py index f164178..baa4c05 100644 --- a/chulk/core/state.py +++ b/chulk/core/state.py @@ -59,13 +59,37 @@ def __post_init__(self) -> None: if self.status not in PLAN_STEP_STATUSES: allowed = ", ".join(sorted(PLAN_STEP_STATUSES)) raise ValueError(f"plan step status must be one of: {allowed}") + if isinstance(self.retry_limit, bool) or not isinstance(self.retry_limit, int): + raise ValueError("plan step retry_limit must be an integer") if self.retry_limit < 0: raise ValueError("plan step retry_limit cannot be negative") - if self.retry_limit != 0: - self.retry_limit = 0 self.depends_on = _dedupe_strings(self.depends_on) self.acceptance_criteria = _dedupe_strings(self.acceptance_criteria) or [self.description] + @property + def retry_count(self) -> int: + """Return the number of step-level recovery attempts already scheduled.""" + count = 0 + for record in self.evidence: + retry_metadata = record.metadata.get("plan_step_retry") + if isinstance(retry_metadata, dict) and retry_metadata.get("disposition") == "retry_scheduled": + count += 1 + return count + + @property + def retries_remaining(self) -> int: + """Return the unused step-level retry budget.""" + return max(0, self.retry_limit - self.retry_count) + + @property + def tool_failure_count(self) -> int: + """Return the number of terminal tool-call failures recorded for this step.""" + return sum( + 1 + for record in self.evidence + if isinstance(record.metadata.get("plan_step_retry"), dict) + ) + def mark(self, status: str) -> None: if status not in PLAN_STEP_STATUSES: allowed = ", ".join(sorted(PLAN_STEP_STATUSES)) @@ -114,6 +138,9 @@ def to_dict(self) -> dict: "depends_on": self.depends_on, "acceptance_criteria": self.acceptance_criteria, "retry_limit": self.retry_limit, + "retry_count": self.retry_count, + "retries_remaining": self.retries_remaining, + "tool_failure_count": self.tool_failure_count, "evidence": [record.to_dict() for record in self.evidence], "started_at": self.started_at, "completed_at": self.completed_at, @@ -184,6 +211,11 @@ def to_prompt(self) -> str: if step.depends_on: lines.append(f" Depends on: {', '.join(step.depends_on)}") lines.append(f" Acceptance criteria: {'; '.join(step.acceptance_criteria)}") + if step.retry_limit: + lines.append( + f" Retry budget: {step.retry_count}/{step.retry_limit} used; " + f"{step.retries_remaining} remaining" + ) if step.evidence: latest = step.evidence[-1] lines.append(f" Evidence: {latest.content}") @@ -198,6 +230,11 @@ def to_user_text(self) -> str: lines = ["Plan", f" summary {self.summary}", " steps"] for step in self.steps: lines.append(f" - [{step.status}] {step.title}: {step.description}") + if step.retry_limit: + lines.append( + f" retries {step.retry_count}/{step.retry_limit} used; " + f"{step.retries_remaining} remaining" + ) if step.evidence: lines.append(f" evidence {step.evidence[-1].content}") if step.blocked_reason: diff --git a/chulk/evals.py b/chulk/evals.py new file mode 100644 index 0000000..52ed109 --- /dev/null +++ b/chulk/evals.py @@ -0,0 +1,283 @@ +"""Deterministic, offline evaluations for Chulk agents. + +The runner always injects :class:`~chulk.testing.ScriptedLLMClient`. Scenarios +therefore exercise the real agent loop, tools, state, and trace callbacks +without reading provider configuration or making network calls. + +Example:: + + from chulk.evals import EvalExpectations, EvalScenario, run_eval + + scenario = EvalScenario( + name="ready", + user_message="Are you ready?", + scripted_responses=({"type": "final_answer", "content": "Ready."},), + expectations=EvalExpectations( + answer="Ready.", + status="completed", + tool_sequence=(), + trace_event_sequence=("model_request_started", "final_answer", "turn_finished"), + ), + ) + run_eval(scenario).assert_passed() + +Trace expectations are matched as an ordered subsequence. This keeps an eval +focused on meaningful lifecycle events while allowing unrelated diagnostics to +be added between them. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any, TypeAlias + +from chulk.core import Agent as CoreAgent +from chulk.testing import ScriptedLLMClient, ScriptedResponse + + +EvalAgentFactory: TypeAlias = Callable[[ScriptedLLMClient], object] + + +@dataclass(frozen=True) +class EvalExpectations: + """Provider-neutral outcomes expected from one agent turn. + + A ``None`` field is not checked. Tool sequences are exact; trace-event + sequences are ordered subsequences of all events emitted by the turn. + """ + + answer: str | None = None + status: str | None = "completed" + tool_sequence: tuple[str, ...] | None = None + trace_event_sequence: tuple[str, ...] | None = None + + def __post_init__(self) -> None: + if self.tool_sequence is not None: + object.__setattr__(self, "tool_sequence", tuple(self.tool_sequence)) + if self.trace_event_sequence is not None: + object.__setattr__(self, "trace_event_sequence", tuple(self.trace_event_sequence)) + + +@dataclass(frozen=True) +class EvalScenario: + """One deterministic user turn and its scripted model actions.""" + + name: str + user_message: str + scripted_responses: tuple[ScriptedResponse, ...] + expectations: EvalExpectations + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("EvalScenario.name cannot be empty") + if not self.user_message.strip(): + raise ValueError("EvalScenario.user_message cannot be empty") + object.__setattr__(self, "scripted_responses", tuple(self.scripted_responses)) + + +@dataclass(frozen=True) +class EvalResult: + """Inspectable actual outcomes and assertion failures for one scenario.""" + + scenario_name: str + passed: bool + answer: str | None + status: str | None + tool_sequence: tuple[str, ...] + trace_event_sequence: tuple[str, ...] + responses_remaining: int + failures: tuple[str, ...] = () + exception: str | None = None + + def assert_passed(self) -> None: + """Raise one readable assertion containing every observed mismatch.""" + if self.passed: + return + details = "\n".join(f"- {failure}" for failure in self.failures) + raise AssertionError(f"Eval scenario {self.scenario_name!r} failed:\n{details}") + + +class EvalRunner: + """Run scenarios with a fresh scripted client and optional agent factory.""" + + def __init__(self, agent_factory: EvalAgentFactory | None = None) -> None: + self.agent_factory = agent_factory + + def run(self, scenario: EvalScenario, *, agent: object | None = None) -> EvalResult: + """Run one scenario against an existing agent or a factory-created one. + + For an existing agent, its model client and trace callback are restored + after the turn. An agent created by this runner is closed after results + have been captured. + """ + if agent is not None and self.agent_factory is not None: + raise ValueError("Pass an agent or configure an agent_factory, not both") + + client = ScriptedLLMClient(scenario.scripted_responses) + instance: object | None = agent + runtime: Any | None = None + owned_agent = agent is None + original_client: object | None = None + original_callback: object | None = None + turn_count = 0 + answer: str | None = None + execution_exception: Exception | None = None + cleanup_exception: Exception | None = None + events: list[str] = [] + + try: + if instance is None: + factory = self.agent_factory or CoreAgent + instance = factory(client) + runtime = _runtime_from_agent(instance) + turn_count = len(runtime.state.turns) + original_client = runtime.llm_client + original_callback = runtime.event_callback + runtime.llm_client = client + + def capture_event(event_type: str, payload: dict[str, Any]) -> None: + events.append(event_type) + if callable(original_callback): + original_callback(event_type, payload) + + runtime.event_callback = capture_event + answer = _run_agent(instance, scenario.user_message) + except Exception as exc: # The result records execution failures for batch evals. + execution_exception = exc + finally: + if runtime is not None: + runtime.event_callback = original_callback + runtime.llm_client = original_client + if owned_agent and instance is not None: + close = getattr(instance, "close", None) + if callable(close): + try: + close() + except Exception as exc: # pragma: no cover - defensive cleanup accounting + cleanup_exception = exc + + turn = _newest_turn(runtime, turn_count) + status = getattr(turn, "status", None) + tool_sequence = _tool_sequence(turn) + failures = _evaluate( + scenario.expectations, + answer=answer, + status=status, + tool_sequence=tool_sequence, + trace_event_sequence=tuple(events), + ) + if execution_exception is not None: + failures.insert(0, f"execution raised {_format_exception(execution_exception)}") + if cleanup_exception is not None: + failures.append(f"agent cleanup raised {_format_exception(cleanup_exception)}") + + return EvalResult( + scenario_name=scenario.name, + passed=not failures, + answer=answer, + status=status, + tool_sequence=tool_sequence, + trace_event_sequence=tuple(events), + responses_remaining=client.remaining, + failures=tuple(failures), + exception=_format_exception(execution_exception) if execution_exception else None, + ) + + def run_many(self, scenarios: Iterable[EvalScenario]) -> tuple[EvalResult, ...]: + """Run each scenario with a fresh agent and return all results.""" + return tuple(self.run(scenario) for scenario in scenarios) + + +def run_eval( + scenario: EvalScenario, + *, + agent: object | None = None, + agent_factory: EvalAgentFactory | None = None, +) -> EvalResult: + """Run one deterministic scenario with a concise functional API.""" + return EvalRunner(agent_factory).run(scenario, agent=agent) + + +def _runtime_from_agent(agent: object) -> Any: + runtime = getattr(agent, "runtime", agent) + required = ("state", "llm_client", "event_callback") + missing = [attribute for attribute in required if not hasattr(runtime, attribute)] + if missing: + raise TypeError(f"Eval agent runtime is missing: {', '.join(missing)}") + return runtime + + +def _run_agent(agent: object, user_message: str) -> str: + run_turn = getattr(agent, "run_turn", None) + if callable(run_turn): + answer = run_turn(user_message) + else: + run = getattr(agent, "run", None) + if not callable(run): + raise TypeError("Eval agent must expose run_turn(message) or run(message)") + answer = run(user_message) + if not isinstance(answer, str): + raise TypeError("Eval agent must return an answer string") + return answer + + +def _newest_turn(runtime: Any | None, previous_count: int) -> Any | None: + if runtime is None: + return None + turns = runtime.state.turns + return turns[-1] if len(turns) > previous_count else None + + +def _tool_sequence(turn: Any | None) -> tuple[str, ...]: + if turn is None: + return () + return tuple(record.resolved_tool_name or record.tool_name for record in turn.tool_calls) + + +def _evaluate( + expected: EvalExpectations, + *, + answer: str | None, + status: str | None, + tool_sequence: tuple[str, ...], + trace_event_sequence: tuple[str, ...], +) -> list[str]: + failures: list[str] = [] + if expected.answer is not None and answer != expected.answer: + failures.append(f"expected answer {expected.answer!r}, got {answer!r}") + if expected.status is not None and status != expected.status: + failures.append(f"expected status {expected.status!r}, got {status!r}") + if expected.tool_sequence is not None and tool_sequence != expected.tool_sequence: + failures.append(f"expected tool sequence {expected.tool_sequence!r}, got {tool_sequence!r}") + if expected.trace_event_sequence is not None and not _is_ordered_subsequence( + expected.trace_event_sequence, + trace_event_sequence, + ): + failures.append( + "expected trace-event subsequence " + f"{expected.trace_event_sequence!r}, got {trace_event_sequence!r}" + ) + return failures + + +def _is_ordered_subsequence(expected: tuple[str, ...], actual: tuple[str, ...]) -> bool: + expected_index = 0 + for event_type in actual: + if expected_index < len(expected) and event_type == expected[expected_index]: + expected_index += 1 + return expected_index == len(expected) + + +def _format_exception(exc: Exception) -> str: + return f"{type(exc).__name__}: {exc}" + + +__all__ = [ + "EvalAgentFactory", + "EvalExpectations", + "EvalResult", + "EvalRunner", + "EvalScenario", + "run_eval", +] diff --git a/chulk/llm/__init__.py b/chulk/llm/__init__.py index f0f8c4c..f2e6c9b 100644 --- a/chulk/llm/__init__.py +++ b/chulk/llm/__init__.py @@ -1,7 +1,11 @@ """LLM provider clients and shared interfaces.""" from chulk.llm.client import ( + AnthropicMessagesClient, + BedrockOpenAICompatibleClient, DeepSeekChatCompletionsClient, + GeminiGenerateContentClient, + HostedOpenAICompatibleClient, LLMActionError, LLMActionResult, LLMClient, @@ -10,25 +14,54 @@ LLMConfigurationError, LLMCost, LLMError, + LLMErrorClassification, + LLMErrorCode, LLMModelCapabilities, LLMProvider, + LLMProviderConnection, + LLMProviderProfile, LLM_PROVIDER_REGISTRY, LLMResponse, LLMStreamChunk, LLMUsage, LocalOpenAICompatibleClient, OpenAIResponsesClient, + OpenRouterChatCompletionsClient, create_llm_client, + provider_capabilities, + provider_connection_from_config, resolve_model_capabilities, supported_llm_providers, ) -from chulk.llm.public import DeepSeekProvider, FallbackChain, FallbackStrategy, LocalProvider, OpenAIProvider, ProviderAttempt +from chulk.llm.capabilities import conservative_model_capabilities +from chulk.llm.public import ( + AnthropicProvider, + BedrockProvider, + BindableLLM, + DeepSeekProvider, + FallbackChain, + FallbackStrategy, + GeminiProvider, + LocalProvider, + OpenAICompatibleProvider, + OpenAIProvider, + OpenRouterProvider, + ProviderAttempt, +) __all__ = [ + "AnthropicMessagesClient", + "AnthropicProvider", + "BedrockOpenAICompatibleClient", + "BedrockProvider", + "BindableLLM", "DeepSeekProvider", "DeepSeekChatCompletionsClient", "FallbackChain", "FallbackStrategy", + "GeminiGenerateContentClient", + "GeminiProvider", + "HostedOpenAICompatibleClient", "LLMActionError", "LLMActionResult", "LLMClient", @@ -37,18 +70,28 @@ "LLMConfigurationError", "LLMCost", "LLMError", + "LLMErrorClassification", + "LLMErrorCode", "LLMModelCapabilities", "LLMProvider", + "LLMProviderConnection", + "LLMProviderProfile", "LLM_PROVIDER_REGISTRY", "LLMResponse", "LLMStreamChunk", "LLMUsage", "LocalOpenAICompatibleClient", "LocalProvider", + "OpenAICompatibleProvider", "OpenAIProvider", "OpenAIResponsesClient", + "OpenRouterChatCompletionsClient", + "OpenRouterProvider", "ProviderAttempt", "create_llm_client", + "conservative_model_capabilities", + "provider_capabilities", + "provider_connection_from_config", "resolve_model_capabilities", "supported_llm_providers", ] diff --git a/chulk/llm/base.py b/chulk/llm/base.py index 1eea099..d37f6e8 100644 --- a/chulk/llm/base.py +++ b/chulk/llm/base.py @@ -2,16 +2,48 @@ from __future__ import annotations +import asyncio from collections.abc import Callable, Iterator from dataclasses import dataclass, field +import inspect import json -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal from chulk.core.actions import ActionParseError, AgentAction, parse_model_response from chulk.core.prompts import JSON_REPAIR_PROMPT from chulk.llm.pricing import estimate_cost from chulk.llm.usage import LLMCost, LLMResponse, LLMUsage, aggregate_cost, aggregate_usage, estimate_usage +if TYPE_CHECKING: + from chulk.llm.capabilities import LLMModelCapabilities + + +LLMErrorCode = Literal[ + "action_shape_error", + "authentication_error", + "configuration_error", + "connection_error", + "fallback_exhausted", + "invalid_request", + "invalid_response", + "model_not_found", + "permission_denied", + "rate_limit", + "server_error", + "timeout", + "unknown", + "unsupported_feature", +] + + +@dataclass(frozen=True) +class LLMErrorClassification: + """Provider-neutral retry and fallback semantics for one failure.""" + + code: LLMErrorCode + retryable: bool + fallback_eligible: bool + class LLMError(RuntimeError): """Base error for model provider failures.""" @@ -22,17 +54,53 @@ def __init__( *, provider: str | None = None, model: str | None = None, - retryable: bool | None = None, + code: LLMErrorCode = "unknown", + retryable: bool = False, + fallback_eligible: bool = False, ) -> None: super().__init__(message) self.provider = provider self.model = model + self.code = code self.retryable = retryable + self.fallback_eligible = fallback_eligible + + @property + def error_code(self) -> LLMErrorCode: + """Compatibility-friendly explicit name for the provider error code.""" + return self.code + + def add_context(self, *, provider: str | None = None, model: str | None = None) -> "LLMError": + """Fill missing provider identity without discarding existing metadata.""" + if self.provider is None: + self.provider = provider + if self.model is None: + self.model = model + return self class LLMConfigurationError(LLMError): """Raised when the LLM client cannot be configured.""" + def __init__( + self, + message: str, + *, + provider: str | None = None, + model: str | None = None, + code: LLMErrorCode = "configuration_error", + retryable: bool = False, + fallback_eligible: bool = False, + ) -> None: + super().__init__( + message, + provider=provider, + model=model, + code=code, + retryable=retryable, + fallback_eligible=fallback_eligible, + ) + class LLMActionError(LLMError): """Raised when an LLM cannot produce a valid agent action.""" @@ -46,8 +114,20 @@ def __init__( raw_response: str | None = None, usage: LLMUsage | None = None, cost: LLMCost | None = None, + provider: str | None = None, + model: str | None = None, + code: LLMErrorCode = "action_shape_error", + retryable: bool = False, + fallback_eligible: bool = False, ) -> None: - super().__init__(message) + super().__init__( + message, + provider=provider, + model=model, + code=code, + retryable=retryable, + fallback_eligible=fallback_eligible, + ) self.repair_attempts = repair_attempts self.errors = errors or [] self.raw_response = raw_response @@ -82,10 +162,21 @@ class LLMStreamChunk: class LLMClient: """Small provider-agnostic LLM client interface.""" + model_capabilities: LLMModelCapabilities | None = None + def complete(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> str: """Return a normal text response.""" raise NotImplementedError + async def acomplete( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> str: + """Return text without blocking the event loop.""" + return (await self.acomplete_response(messages, max_output_tokens=max_output_tokens)).content + def complete_response( self, messages: list[dict[str, str]], @@ -93,15 +184,30 @@ def complete_response( max_output_tokens: int | None = None, ) -> LLMResponse: """Return text plus normalized usage metadata.""" - try: - if max_output_tokens is None: - content = self.complete(messages) - else: - content = self.complete(messages, max_output_tokens=max_output_tokens) - except TypeError: - content = self.complete(messages) + kwargs = {"max_output_tokens": max_output_tokens} if max_output_tokens is not None else {} + content = call_with_supported_kwargs(self.complete, messages, **kwargs) return self._response_with_estimated_usage(messages, content) + async def acomplete_response( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + """Return text and usage through a native async or sync compatibility path. + + Provider clients override this method when their SDK exposes a native + async transport. Sync-only injected clients run in asyncio's bounded + default executor so they do not block the caller's event loop. + """ + kwargs = {"max_output_tokens": max_output_tokens} if max_output_tokens is not None else {} + return await asyncio.to_thread( + call_with_supported_kwargs, + self.complete_response, + messages, + **kwargs, + ) + def stream_complete( self, messages: list[dict[str, str]], @@ -120,14 +226,11 @@ def stream_complete( def complete_json(self, messages: list[dict[str, str]]) -> dict[str, Any]: """Return a structured JSON response.""" - raw_response = self.complete(messages) - try: - parsed = json.loads(raw_response) - except json.JSONDecodeError as exc: - raise LLMError("Model response was not valid JSON") from exc - if not isinstance(parsed, dict): - raise LLMError("Model JSON response must be an object") - return parsed + return _parse_json_object(self.complete(messages), client=self) + + async def acomplete_json(self, messages: list[dict[str, str]]) -> dict[str, Any]: + """Return a structured JSON response without blocking the event loop.""" + return _parse_json_object(await self.acomplete(messages), client=self) def complete_action( self, @@ -150,33 +253,111 @@ def complete_action( usage_records: list[LLMUsage | None] = [] cost_records: list[LLMCost | None] = [] for attempt in range(max_repair_attempts + 1): - if max_output_tokens is None: - response = self._complete_action_response_once( - action_messages, - tools=tools, - hosted_mcp_servers=hosted_mcp_servers, - mcp_approval_callback=mcp_approval_callback, - ) - else: - response = self._complete_action_response_once( - action_messages, - max_output_tokens=max_output_tokens, - tools=tools, - hosted_mcp_servers=hosted_mcp_servers, - mcp_approval_callback=mcp_approval_callback, + response_kwargs: dict[str, Any] = { + "tools": tools, + "hosted_mcp_servers": hosted_mcp_servers, + "mcp_approval_callback": mcp_approval_callback, + } + if max_output_tokens is not None: + response_kwargs["max_output_tokens"] = max_output_tokens + response = call_with_supported_kwargs( + self._complete_action_response_once, + action_messages, + **response_kwargs, + ) + try: + return _parse_action_response( + response, + attempt=attempt, + errors=errors, + usage_records=usage_records, + cost_records=cost_records, ) - raw_response = response.content - usage_records.append(response.usage) - cost_records.append(response.cost) + except ActionParseError as exc: + errors.append(str(exc)) + if attempt >= max_repair_attempts: + raise LLMActionError( + f"Model response was not valid action JSON: {exc}", + repair_attempts=attempt, + errors=errors, + raw_response=response.content, + usage=aggregate_usage(usage_records), + cost=aggregate_cost(cost_records), + provider=_provider_name(self), + model=_model_name(self), + ) from exc + action_messages = [ + *action_messages, + { + "role": "user", + "content": _format_json_repair_prompt(response.content, str(exc)), + }, + ] + + raise LLMActionError( + "Model response was not valid action JSON", + provider=_provider_name(self), + model=_model_name(self), + ) + + async def acomplete_action( + self, + messages: list[dict[str, str]], + *, + max_repair_attempts: int = 2, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, + ) -> LLMActionResult: + """Return a validated action through the client's async transport.""" + if ( + type(self).complete_action is not LLMClient.complete_action + and type(self).acomplete_action is LLMClient.acomplete_action + ): + compatibility_kwargs: dict[str, Any] = { + "max_repair_attempts": max_repair_attempts, + "tools": tools, + "hosted_mcp_servers": hosted_mcp_servers, + "mcp_approval_callback": mcp_approval_callback, + } + if max_output_tokens is not None: + compatibility_kwargs["max_output_tokens"] = max_output_tokens + return await asyncio.to_thread( + call_with_supported_kwargs, + self.complete_action, + messages, + **compatibility_kwargs, + ) + if max_repair_attempts < 0: + raise ValueError("max_repair_attempts cannot be negative") + if max_output_tokens is not None and max_output_tokens < 1: + raise ValueError("max_output_tokens must be greater than zero") + + action_messages = list(messages) + errors: list[str] = [] + usage_records: list[LLMUsage | None] = [] + cost_records: list[LLMCost | None] = [] + for attempt in range(max_repair_attempts + 1): + response_kwargs: dict[str, Any] = { + "tools": tools, + "hosted_mcp_servers": hosted_mcp_servers, + "mcp_approval_callback": mcp_approval_callback, + } + if max_output_tokens is not None: + response_kwargs["max_output_tokens"] = max_output_tokens + response = await call_async_with_supported_kwargs( + self._acomplete_action_response_once, + action_messages, + **response_kwargs, + ) try: - return LLMActionResult( - action=parse_model_response(raw_response), - raw_response=raw_response, - repair_attempts=attempt, + return _parse_action_response( + response, + attempt=attempt, errors=errors, - usage=aggregate_usage(usage_records), - cost=aggregate_cost(cost_records), - metadata=response.metadata, + usage_records=usage_records, + cost_records=cost_records, ) except ActionParseError as exc: errors.append(str(exc)) @@ -185,19 +366,25 @@ def complete_action( f"Model response was not valid action JSON: {exc}", repair_attempts=attempt, errors=errors, - raw_response=raw_response, + raw_response=response.content, usage=aggregate_usage(usage_records), cost=aggregate_cost(cost_records), + provider=_provider_name(self), + model=_model_name(self), ) from exc action_messages = [ *action_messages, { "role": "user", - "content": _format_json_repair_prompt(raw_response, str(exc)), + "content": _format_json_repair_prompt(response.content, str(exc)), }, ] - raise LLMActionError("Model response was not valid action JSON") + raise LLMActionError( + "Model response was not valid action JSON", + provider=_provider_name(self), + model=_model_name(self), + ) def _complete_action_once(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> str: """Return one raw action response attempt.""" @@ -213,15 +400,34 @@ def _complete_action_response_once( mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, ) -> LLMResponse: """Return one raw action response attempt plus metadata.""" - try: - if max_output_tokens is None: - content = self._complete_action_once(messages) - else: - content = self._complete_action_once(messages, max_output_tokens=max_output_tokens) - except TypeError: - content = self._complete_action_once(messages) + kwargs = {"max_output_tokens": max_output_tokens} if max_output_tokens is not None else {} + content = call_with_supported_kwargs(self._complete_action_once, messages, **kwargs) return self._response_with_estimated_usage(messages, content) + async def _acomplete_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, + ) -> LLMResponse: + """Compatibility hook for sync-only custom clients.""" + kwargs: dict[str, Any] = { + "tools": tools, + "hosted_mcp_servers": hosted_mcp_servers, + "mcp_approval_callback": mcp_approval_callback, + } + if max_output_tokens is not None: + kwargs["max_output_tokens"] = max_output_tokens + return await asyncio.to_thread( + call_with_supported_kwargs, + self._complete_action_response_once, + messages, + **kwargs, + ) + def _response_with_estimated_usage(self, messages: list[dict[str, str]], content: str) -> LLMResponse: provider = _provider_name(self) model = _model_name(self) @@ -235,6 +441,49 @@ def _response_with_estimated_usage(self, messages: list[dict[str, str]], content ) +def _parse_json_object(raw_response: str, *, client: LLMClient) -> dict[str, Any]: + """Parse a provider response as one JSON object.""" + try: + parsed = json.loads(raw_response) + except json.JSONDecodeError as exc: + raise LLMError( + "Model response was not valid JSON", + provider=_provider_name(client), + model=_model_name(client), + code="action_shape_error", + ) from exc + if not isinstance(parsed, dict): + raise LLMError( + "Model JSON response must be an object", + provider=_provider_name(client), + model=_model_name(client), + code="action_shape_error", + ) + return parsed + + +def _parse_action_response( + response: LLMResponse, + *, + attempt: int, + errors: list[str], + usage_records: list[LLMUsage | None], + cost_records: list[LLMCost | None], +) -> LLMActionResult: + """Parse one sync or async action response and aggregate its accounting.""" + usage_records.append(response.usage) + cost_records.append(response.cost) + return LLMActionResult( + action=parse_model_response(response.content), + raw_response=response.content, + repair_attempts=attempt, + errors=errors, + usage=aggregate_usage(usage_records), + cost=aggregate_cost(cost_records), + metadata=response.metadata, + ) + + def _provider_name(client: object) -> str | None: value = getattr(client, "provider", None) or getattr(client, "name", None) return str(value) if value is not None else None @@ -245,6 +494,141 @@ def _model_name(client: object) -> str | None: return str(value) if value is not None else None +def provider_error_from_exception( + exc: Exception, + *, + message: str, + provider: str, + model: str | None, + action_transport: bool = False, +) -> LLMError: + """Normalize an SDK exception while preserving an existing Chulk error.""" + if isinstance(exc, LLMError): + return exc.add_context(provider=provider, model=model) + classification = classify_provider_exception(exc, action_transport=action_transport) + return LLMError( + f"{message}: {exc}", + provider=provider, + model=model, + code=classification.code, + retryable=classification.retryable, + fallback_eligible=classification.fallback_eligible, + ) + + +def classify_provider_exception(exc: Exception, *, action_transport: bool = False) -> LLMErrorClassification: + """Classify OpenAI-style SDK failures without requiring the SDK at import time.""" + class_names = {item.__name__ for item in type(exc).__mro__} + status_code = _exception_status_code(exc) + provider_code = _exception_provider_code(exc) + message = str(exc).lower() + + if "AuthenticationError" in class_names or status_code == 401 or provider_code in { + "authentication_error", + "invalid_api_key", + }: + return LLMErrorClassification("authentication_error", retryable=False, fallback_eligible=False) + if "PermissionDeniedError" in class_names or status_code == 403: + return LLMErrorClassification("permission_denied", retryable=False, fallback_eligible=False) + if "RateLimitError" in class_names or status_code == 429: + return LLMErrorClassification("rate_limit", retryable=True, fallback_eligible=True) + if _has_timeout_class(class_names) or isinstance(exc, TimeoutError) or status_code == 408: + return LLMErrorClassification("timeout", retryable=True, fallback_eligible=True) + if _has_connection_class(class_names) or isinstance(exc, ConnectionError): + return LLMErrorClassification("connection_error", retryable=True, fallback_eligible=True) + if "InternalServerError" in class_names or (status_code is not None and status_code >= 500): + return LLMErrorClassification("server_error", retryable=True, fallback_eligible=True) + if "NotFoundError" in class_names or _is_model_error(provider_code, message, status_code): + return LLMErrorClassification("model_not_found", retryable=False, fallback_eligible=False) + if action_transport and _is_unsupported_action_transport(provider_code, message): + return LLMErrorClassification("unsupported_feature", retryable=False, fallback_eligible=True) + if ( + "BadRequestError" in class_names + or "UnprocessableEntityError" in class_names + or status_code in {400, 404, 409, 422} + ): + return LLMErrorClassification("invalid_request", retryable=False, fallback_eligible=False) + return LLMErrorClassification("unknown", retryable=False, fallback_eligible=False) + + +def is_action_transport_fallback_error(exc: Exception) -> bool: + """Return whether native tool calling may safely retry via Chulk action JSON.""" + return isinstance(exc, LLMError) and exc.code in {"action_shape_error", "unsupported_feature"} + + +def _exception_status_code(exc: Exception) -> int | None: + value = getattr(exc, "status_code", None) + if isinstance(value, int): + return value + response = getattr(exc, "response", None) + value = getattr(response, "status_code", None) + return value if isinstance(value, int) else None + + +def _exception_provider_code(exc: Exception) -> str | None: + value = getattr(exc, "code", None) + if isinstance(value, str) and value: + return value.lower() + body = getattr(exc, "body", None) + if isinstance(body, dict): + error = body.get("error", body) + if isinstance(error, dict): + value = error.get("code") + if isinstance(value, str) and value: + return value.lower() + return None + + +def _is_model_error(provider_code: str | None, message: str, status_code: int | None) -> bool: + if provider_code in {"model_not_found", "invalid_model", "unknown_model"}: + return True + if status_code == 404: + return True + model_markers = ("model not found", "model_not_found", "unknown model", "does not exist") + return any(marker in message for marker in model_markers) and status_code in {None, 400, 404, 422} + + +def _is_unsupported_action_transport(provider_code: str | None, message: str) -> bool: + if provider_code in {"unsupported_feature", "unsupported_parameter"}: + return True + unsupported_markers = ( + "does not support tool", + "doesn't support tool", + "native tools unsupported", + "tool calling is not supported", + "tool calls are not supported", + "tools unsupported", + "unsupported parameter: tools", + "unsupported_parameter: tools", + ) + return any(marker in message for marker in unsupported_markers) + + +def _has_timeout_class(class_names: set[str]) -> bool: + """Recognize common SDK and HTTP-client timeout families by class name.""" + known_names = { + "APITimeoutError", + "ConnectTimeout", + "PoolTimeout", + "ReadTimeout", + "TimeoutException", + "WriteTimeout", + } + return bool(class_names & known_names) or any(name.endswith("TimeoutError") for name in class_names) + + +def _has_connection_class(class_names: set[str]) -> bool: + """Recognize common SDK and HTTP-client connection families by class name.""" + known_names = { + "APIConnectionError", + "ConnectError", + "NetworkError", + "ProxyError", + "RemoteProtocolError", + } + return bool(class_names & known_names) or any(name.endswith("ConnectionError") for name in class_names) + + def _format_json_repair_prompt(raw_response: str, error: str) -> str: return "\n".join( [ @@ -254,3 +638,45 @@ def _format_json_repair_prompt(raw_response: str, error: str) -> str: raw_response[:2000], ] ) + + +def call_with_supported_kwargs(call: Callable[..., Any], /, *args: Any, **kwargs: Any) -> Any: + """Call a compatibility hook once, omitting only unsupported keyword arguments. + + Older injected clients may implement the original, smaller Chulk method + signatures. Inspecting the callable before invocation preserves that + compatibility without catching an internal ``TypeError`` and accidentally + issuing the same provider request twice. + """ + supported_kwargs = _supported_kwargs(call, kwargs) + return call(*args, **supported_kwargs) + + +async def call_async_with_supported_kwargs( + call: Callable[..., Any], + /, + *args: Any, + **kwargs: Any, +) -> Any: + """Await a compatibility hook once, omitting unsupported keywords.""" + result = call(*args, **_supported_kwargs(call, kwargs)) + if not inspect.isawaitable(result): + raise TypeError(f"Async compatibility hook returned a non-awaitable: {call!r}") + return await result + + +def _supported_kwargs(call: Callable[..., Any], kwargs: dict[str, Any]) -> dict[str, Any]: + if not kwargs: + return {} + try: + parameters = inspect.signature(call).parameters.values() + except (TypeError, ValueError): + return dict(kwargs) + if any(parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters): + return dict(kwargs) + accepted = { + parameter.name + for parameter in parameters + if parameter.kind in {inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY} + } + return {name: value for name, value in kwargs.items() if name in accepted} diff --git a/chulk/llm/capabilities.py b/chulk/llm/capabilities.py index f238cd0..cc82497 100644 --- a/chulk/llm/capabilities.py +++ b/chulk/llm/capabilities.py @@ -13,6 +13,14 @@ LOCAL_DEFAULT_CONTEXT_WINDOW_TOKENS = 131_072 LOCAL_DEFAULT_RESPONSE_RESERVE_TOKENS = 4_096 LOCAL_QWEN_3_5_35B_CONTEXT_WINDOW_TOKENS = 262_144 +COMPATIBLE_DEFAULT_CONTEXT_WINDOW_TOKENS = 8_192 +COMPATIBLE_DEFAULT_RESPONSE_RESERVE_TOKENS = 2_048 +ANTHROPIC_DEFAULT_CONTEXT_WINDOW_TOKENS = 200_000 +ANTHROPIC_DEFAULT_RESPONSE_RESERVE_TOKENS = 4_096 +BEDROCK_DEFAULT_CONTEXT_WINDOW_TOKENS = 8_192 +BEDROCK_DEFAULT_RESPONSE_RESERVE_TOKENS = 2_048 +GEMINI_DEFAULT_CONTEXT_WINDOW_TOKENS = 1_048_576 +GEMINI_DEFAULT_RESPONSE_RESERVE_TOKENS = 8_192 OPENAI_GPT_4_1_LIMITS = ( OPENAI_GPT_4_1_CONTEXT_WINDOW_TOKENS, @@ -30,6 +38,22 @@ LOCAL_QWEN_3_5_35B_CONTEXT_WINDOW_TOKENS, LOCAL_DEFAULT_RESPONSE_RESERVE_TOKENS, ) +COMPATIBLE_DEFAULT_LIMITS = ( + COMPATIBLE_DEFAULT_CONTEXT_WINDOW_TOKENS, + COMPATIBLE_DEFAULT_RESPONSE_RESERVE_TOKENS, +) +ANTHROPIC_DEFAULT_LIMITS = ( + ANTHROPIC_DEFAULT_CONTEXT_WINDOW_TOKENS, + ANTHROPIC_DEFAULT_RESPONSE_RESERVE_TOKENS, +) +BEDROCK_DEFAULT_LIMITS = ( + BEDROCK_DEFAULT_CONTEXT_WINDOW_TOKENS, + BEDROCK_DEFAULT_RESPONSE_RESERVE_TOKENS, +) +GEMINI_DEFAULT_LIMITS = ( + GEMINI_DEFAULT_CONTEXT_WINDOW_TOKENS, + GEMINI_DEFAULT_RESPONSE_RESERVE_TOKENS, +) @dataclass(frozen=True) @@ -41,7 +65,12 @@ class LLMCapabilities: supports_streaming: bool = False supports_native_tool_calling: bool = False supports_hosted_mcp_tools: bool = False - api_style: Literal["responses", "chat_completions"] = "chat_completions" + api_style: Literal[ + "responses", + "chat_completions", + "messages", + "generate_content", + ] = "chat_completions" @dataclass(frozen=True) @@ -82,6 +111,27 @@ def register_model_capabilities(capabilities: LLMModelCapabilities) -> None: MODEL_CAPABILITIES[key] = capabilities +def conservative_model_capabilities( + capabilities: list[LLMModelCapabilities] | tuple[LLMModelCapabilities, ...], +) -> LLMModelCapabilities: + """Return limits that are safe for every model in a fallback path.""" + if not capabilities: + raise ValueError("At least one model capability record is required") + context_window = min(item.context_window_tokens for item in capabilities) + response_reserve = min( + max(item.default_response_reserve_tokens for item in capabilities), + context_window, + ) + providers = ",".join(dict.fromkeys(item.provider for item in capabilities)) + models = ",".join(dict.fromkeys(item.model for item in capabilities)) + return LLMModelCapabilities( + provider=providers, + model=models, + context_window_tokens=context_window, + default_response_reserve_tokens=response_reserve, + ) + + def resolve_model_capabilities(provider: str, model: str) -> LLMModelCapabilities: """Return required token metadata for a configured model.""" key = _model_key(provider, model) @@ -108,6 +158,34 @@ def _model_key(provider: str, model: str) -> tuple[str, str]: def _resolve_model_family_capabilities(provider: str, model: str) -> LLMModelCapabilities | None: normalized_provider, normalized_model = _model_key(provider, model) + if normalized_provider == "anthropic": + return LLMModelCapabilities( + provider=normalized_provider, + model=normalized_model, + context_window_tokens=ANTHROPIC_DEFAULT_CONTEXT_WINDOW_TOKENS, + default_response_reserve_tokens=ANTHROPIC_DEFAULT_RESPONSE_RESERVE_TOKENS, + ) + if normalized_provider == "bedrock": + return LLMModelCapabilities( + provider=normalized_provider, + model=normalized_model, + context_window_tokens=BEDROCK_DEFAULT_CONTEXT_WINDOW_TOKENS, + default_response_reserve_tokens=BEDROCK_DEFAULT_RESPONSE_RESERVE_TOKENS, + ) + if normalized_provider == "gemini": + return LLMModelCapabilities( + provider=normalized_provider, + model=normalized_model, + context_window_tokens=GEMINI_DEFAULT_CONTEXT_WINDOW_TOKENS, + default_response_reserve_tokens=GEMINI_DEFAULT_RESPONSE_RESERVE_TOKENS, + ) + if normalized_provider in {"openai-compatible", "openrouter"}: + return LLMModelCapabilities( + provider=normalized_provider, + model=normalized_model, + context_window_tokens=COMPATIBLE_DEFAULT_CONTEXT_WINDOW_TOKENS, + default_response_reserve_tokens=COMPATIBLE_DEFAULT_RESPONSE_RESERVE_TOKENS, + ) if normalized_provider == "local": return LLMModelCapabilities( provider=normalized_provider, diff --git a/chulk/llm/client.py b/chulk/llm/client.py index 15d42f3..d2024a5 100644 --- a/chulk/llm/client.py +++ b/chulk/llm/client.py @@ -6,6 +6,8 @@ LLMClient, LLMConfigurationError, LLMError, + LLMErrorClassification, + LLMErrorCode, LLMStreamChunk, ) from chulk.llm.capabilities import LLMCapabilities, LLMModelCapabilities, resolve_model_capabilities @@ -13,16 +15,31 @@ LLM_PROVIDER_REGISTRY, LLMClientSettings, LLMProvider, + LLMProviderConnection, + LLMProviderProfile, create_llm_client, + provider_capabilities, + provider_connection_from_config, supported_llm_providers, ) +from chulk.llm.providers.anthropic import AnthropicMessagesClient +from chulk.llm.providers.bedrock import BedrockOpenAICompatibleClient from chulk.llm.providers.deepseek import DeepSeekChatCompletionsClient +from chulk.llm.providers.gemini import GeminiGenerateContentClient +from chulk.llm.providers.compatible import ( + HostedOpenAICompatibleClient, + OpenRouterChatCompletionsClient, +) from chulk.llm.providers.local import LocalOpenAICompatibleClient from chulk.llm.providers.openai import OpenAIResponsesClient from chulk.llm.usage import LLMCost, LLMResponse, LLMUsage __all__ = [ + "AnthropicMessagesClient", + "BedrockOpenAICompatibleClient", "DeepSeekChatCompletionsClient", + "HostedOpenAICompatibleClient", + "GeminiGenerateContentClient", "LLMActionError", "LLMActionResult", "LLMClient", @@ -31,15 +48,22 @@ "LLMConfigurationError", "LLMCost", "LLMError", + "LLMErrorClassification", + "LLMErrorCode", "LLMModelCapabilities", "LLMProvider", + "LLMProviderConnection", + "LLMProviderProfile", "LLM_PROVIDER_REGISTRY", "LLMResponse", "LLMStreamChunk", "LLMUsage", "LocalOpenAICompatibleClient", "OpenAIResponsesClient", + "OpenRouterChatCompletionsClient", "create_llm_client", + "provider_capabilities", + "provider_connection_from_config", "resolve_model_capabilities", "supported_llm_providers", ] diff --git a/chulk/llm/factory.py b/chulk/llm/factory.py index 6cf73f4..eb9d74e 100644 --- a/chulk/llm/factory.py +++ b/chulk/llm/factory.py @@ -1,56 +1,254 @@ -"""LLM provider registry and client factory.""" +"""LLM provider profiles and client construction.""" from __future__ import annotations from collections.abc import Callable from dataclasses import dataclass +from typing import Protocol from chulk.llm.base import LLMClient, LLMConfigurationError from chulk.llm.capabilities import LLMCapabilities, resolve_model_capabilities -from chulk.llm.providers.deepseek import DEEPSEEK_CAPABILITIES, DeepSeekChatCompletionsClient -from chulk.llm.providers.local import DEFAULT_LOCAL_BASE_URL, LOCAL_CAPABILITIES, LocalOpenAICompatibleClient +from chulk.llm.providers.anthropic import ANTHROPIC_CAPABILITIES, AnthropicMessagesClient +from chulk.llm.providers.bedrock import BEDROCK_CAPABILITIES, BedrockOpenAICompatibleClient +from chulk.llm.providers.compatible import ( + DEFAULT_OPENROUTER_BASE_URL, + HOSTED_OPENAI_COMPATIBLE_CAPABILITIES, + OPENROUTER_CAPABILITIES, + HostedOpenAICompatibleClient, + OpenRouterChatCompletionsClient, +) +from chulk.llm.providers.deepseek import ( + DEFAULT_DEEPSEEK_BASE_URL, + DEEPSEEK_CAPABILITIES, + DeepSeekChatCompletionsClient, +) +from chulk.llm.providers.gemini import GEMINI_CAPABILITIES, GeminiGenerateContentClient +from chulk.llm.providers.local import ( + DEFAULT_LOCAL_BASE_URL, + LOCAL_CAPABILITIES, + LocalOpenAICompatibleClient, +) from chulk.llm.providers.openai import OPENAI_CAPABILITIES, OpenAIResponsesClient +class LLMConnectionConfig(Protocol): + """Runtime configuration fields used to bind built-in providers.""" + + @property + def openai_api_key(self) -> str | None: ... + + @property + def deepseek_api_key(self) -> str | None: ... + + @property + def deepseek_base_url(self) -> str: ... + + @property + def local_api_key(self) -> str | None: ... + + @property + def local_base_url(self) -> str: ... + + @property + def openai_compatible_api_key(self) -> str | None: ... + + @property + def openai_compatible_base_url(self) -> str | None: ... + + @property + def openrouter_api_key(self) -> str | None: ... + + @property + def openrouter_base_url(self) -> str: ... + + @property + def anthropic_api_key(self) -> str | None: ... + + @property + def anthropic_base_url(self) -> str | None: ... + + @property + def bedrock_api_key(self) -> str | None: ... + + @property + def bedrock_base_url(self) -> str | None: ... + + @property + def gemini_api_key(self) -> str | None: ... + + @property + def gemini_base_url(self) -> str | None: ... + + @dataclass(frozen=True) +class LLMProviderConnection: + """Connection details for one selected provider.""" + + api_key: str | None = None + base_url: str | None = None + + def with_overrides( + self, + *, + api_key: str | None = None, + base_url: str | None = None, + ) -> "LLMProviderConnection": + """Return a connection with explicit non-None overrides applied.""" + return LLMProviderConnection( + api_key=self.api_key if api_key is None else api_key, + base_url=self.base_url if base_url is None else base_url, + ) + + +@dataclass(frozen=True, init=False) class LLMClientSettings: - """Provider-neutral settings used to construct an LLM client.""" + """Settings passed to registered LLM client factories. + + ``connection`` is the provider-neutral path used by built-in providers. + The original credential fields remain available so third-party provider + callbacks written against the first public factory contract keep working. + """ model: str + connection: LLMProviderConnection + timeout_seconds: float + max_retries: int openai_api_key: str | None deepseek_api_key: str | None deepseek_base_url: str local_api_key: str | None local_base_url: str - timeout_seconds: float - max_retries: int + + def __init__( + self, + model: str, + openai_api_key: str | None = None, + deepseek_api_key: str | None = None, + deepseek_base_url: str = DEFAULT_DEEPSEEK_BASE_URL, + local_api_key: str | None = None, + local_base_url: str = DEFAULT_LOCAL_BASE_URL, + timeout_seconds: float = 60.0, + max_retries: int = 2, + *, + connection: LLMProviderConnection | None = None, + ) -> None: + """Accept both the original positional fields and a bound connection.""" + object.__setattr__(self, "model", model) + object.__setattr__( + self, + "connection", + connection if connection is not None else LLMProviderConnection(), + ) + object.__setattr__(self, "timeout_seconds", timeout_seconds) + object.__setattr__(self, "max_retries", max_retries) + object.__setattr__(self, "openai_api_key", openai_api_key) + object.__setattr__(self, "deepseek_api_key", deepseek_api_key) + object.__setattr__(self, "deepseek_base_url", deepseek_base_url) + object.__setattr__(self, "local_api_key", local_api_key) + object.__setattr__(self, "local_base_url", local_base_url) @dataclass(frozen=True) -class LLMProvider: - """Factory metadata for one configured LLM provider.""" +class LLMProviderProfile: + """Factory, connection, and capability metadata for one provider.""" name: str capabilities: LLMCapabilities create_client: Callable[[LLMClientSettings], LLMClient] + default_connection: LLMProviderConnection = LLMProviderConnection() + connection_from_config: Callable[[LLMConnectionConfig], LLMProviderConnection] | None = None + + def bind_connection(self, config: LLMConnectionConfig) -> LLMProviderConnection: + """Resolve this provider's connection from the shared runtime config.""" + configured = ( + self.connection_from_config(config) + if self.connection_from_config is not None + else self.default_connection + ) + return self.default_connection.with_overrides( + api_key=configured.api_key, + base_url=configured.base_url, + ) + + +# Compatibility alias retained for callers that imported the original registry type. +LLMProvider = LLMProviderProfile -LLM_PROVIDER_REGISTRY: dict[str, LLMProvider] = { - "openai": LLMProvider( +LLM_PROVIDER_REGISTRY: dict[str, LLMProviderProfile] = { + "openai": LLMProviderProfile( name="openai", capabilities=OPENAI_CAPABILITIES, + default_connection=LLMProviderConnection(), + connection_from_config=lambda config: LLMProviderConnection(api_key=config.openai_api_key), create_client=lambda settings: _create_openai_client(settings), ), - "deepseek": LLMProvider( + "deepseek": LLMProviderProfile( name="deepseek", capabilities=DEEPSEEK_CAPABILITIES, + default_connection=LLMProviderConnection(base_url=DEFAULT_DEEPSEEK_BASE_URL), + connection_from_config=lambda config: LLMProviderConnection( + api_key=config.deepseek_api_key, + base_url=config.deepseek_base_url, + ), create_client=lambda settings: _create_deepseek_client(settings), ), - "local": LLMProvider( + "local": LLMProviderProfile( name="local", capabilities=LOCAL_CAPABILITIES, + default_connection=LLMProviderConnection(base_url=DEFAULT_LOCAL_BASE_URL), + connection_from_config=lambda config: LLMProviderConnection( + api_key=config.local_api_key, + base_url=config.local_base_url, + ), create_client=lambda settings: _create_local_client(settings), ), + "openai-compatible": LLMProviderProfile( + name="openai-compatible", + capabilities=HOSTED_OPENAI_COMPATIBLE_CAPABILITIES, + connection_from_config=lambda config: LLMProviderConnection( + api_key=config.openai_compatible_api_key, + base_url=config.openai_compatible_base_url, + ), + create_client=lambda settings: _create_hosted_compatible_client(settings), + ), + "openrouter": LLMProviderProfile( + name="openrouter", + capabilities=OPENROUTER_CAPABILITIES, + default_connection=LLMProviderConnection(base_url=DEFAULT_OPENROUTER_BASE_URL), + connection_from_config=lambda config: LLMProviderConnection( + api_key=config.openrouter_api_key, + base_url=config.openrouter_base_url, + ), + create_client=lambda settings: _create_openrouter_client(settings), + ), + "anthropic": LLMProviderProfile( + name="anthropic", + capabilities=ANTHROPIC_CAPABILITIES, + connection_from_config=lambda config: LLMProviderConnection( + api_key=config.anthropic_api_key, + base_url=config.anthropic_base_url, + ), + create_client=lambda settings: _create_anthropic_client(settings), + ), + "bedrock": LLMProviderProfile( + name="bedrock", + capabilities=BEDROCK_CAPABILITIES, + connection_from_config=lambda config: LLMProviderConnection( + api_key=config.bedrock_api_key, + base_url=config.bedrock_base_url, + ), + create_client=lambda settings: _create_bedrock_client(settings), + ), + "gemini": LLMProviderProfile( + name="gemini", + capabilities=GEMINI_CAPABILITIES, + connection_from_config=lambda config: LLMProviderConnection( + api_key=config.gemini_api_key, + base_url=config.gemini_base_url, + ), + create_client=lambda settings: _create_gemini_client(settings), + ), } @@ -59,46 +257,167 @@ def supported_llm_providers() -> set[str]: return set(LLM_PROVIDER_REGISTRY) +def provider_capabilities(provider: str) -> LLMCapabilities: + """Return capability metadata for one registered provider.""" + return _provider_profile(provider).capabilities + + +def provider_connection_from_config( + provider: str, + config: LLMConnectionConfig, +) -> LLMProviderConnection: + """Resolve one registered provider connection from runtime config.""" + return _provider_profile(provider).bind_connection(config) + + def create_llm_client( *, provider: str, model: str, - openai_api_key: str | None, - deepseek_api_key: str | None, - deepseek_base_url: str, timeout_seconds: float, max_retries: int, + connection: LLMProviderConnection | None = None, + openai_api_key: str | None = None, + deepseek_api_key: str | None = None, + deepseek_base_url: str | None = None, local_api_key: str | None = None, - local_base_url: str = DEFAULT_LOCAL_BASE_URL, + local_base_url: str | None = None, + openai_compatible_api_key: str | None = None, + openai_compatible_base_url: str | None = None, + openrouter_api_key: str | None = None, + openrouter_base_url: str | None = None, + anthropic_api_key: str | None = None, + anthropic_base_url: str | None = None, + bedrock_api_key: str | None = None, + bedrock_base_url: str | None = None, + gemini_api_key: str | None = None, + gemini_base_url: str | None = None, ) -> LLMClient: - """Create an LLM client for the selected provider.""" + """Create an LLM client for the selected provider. + + ``connection`` is the provider-neutral construction path. The named + credential arguments remain accepted for compatibility with existing SDK + callers and are normalized into the same connection object. + """ normalized_provider = provider.lower() - provider_spec = LLM_PROVIDER_REGISTRY.get(normalized_provider) - if provider_spec is None: - raise LLMConfigurationError(f"Unsupported LLM provider: {provider}") + provider_profile = _provider_profile(normalized_provider) try: - resolve_model_capabilities(normalized_provider, model) + model_capabilities = resolve_model_capabilities(normalized_provider, model) except ValueError as exc: raise LLMConfigurationError(str(exc)) from exc - return provider_spec.create_client( + selected_connection = connection or _legacy_connection( + provider_profile, + openai_api_key=openai_api_key, + deepseek_api_key=deepseek_api_key, + deepseek_base_url=deepseek_base_url, + local_api_key=local_api_key, + local_base_url=local_base_url, + openai_compatible_api_key=openai_compatible_api_key, + openai_compatible_base_url=openai_compatible_base_url, + openrouter_api_key=openrouter_api_key, + openrouter_base_url=openrouter_base_url, + anthropic_api_key=anthropic_api_key, + anthropic_base_url=anthropic_base_url, + bedrock_api_key=bedrock_api_key, + bedrock_base_url=bedrock_base_url, + gemini_api_key=gemini_api_key, + gemini_base_url=gemini_base_url, + ) + selected_connection = provider_profile.default_connection.with_overrides( + api_key=selected_connection.api_key, + base_url=selected_connection.base_url, + ) + client = provider_profile.create_client( LLMClientSettings( model=model, - openai_api_key=openai_api_key, - deepseek_api_key=deepseek_api_key, - deepseek_base_url=deepseek_base_url, - local_api_key=local_api_key, - local_base_url=local_base_url, + openai_api_key=( + openai_api_key + if openai_api_key is not None + else selected_connection.api_key + ), + deepseek_api_key=( + deepseek_api_key + if deepseek_api_key is not None + else selected_connection.api_key + ), + deepseek_base_url=( + deepseek_base_url + if deepseek_base_url is not None + else selected_connection.base_url or DEFAULT_DEEPSEEK_BASE_URL + ), + local_api_key=( + local_api_key + if local_api_key is not None + else selected_connection.api_key + ), + local_base_url=( + local_base_url + if local_base_url is not None + else selected_connection.base_url or DEFAULT_LOCAL_BASE_URL + ), + connection=selected_connection, timeout_seconds=timeout_seconds, max_retries=max_retries, ) ) + client.model_capabilities = model_capabilities + return client + + +def _provider_profile(provider: str) -> LLMProviderProfile: + normalized_provider = provider.lower() + provider_profile = LLM_PROVIDER_REGISTRY.get(normalized_provider) + if provider_profile is None: + raise LLMConfigurationError(f"Unsupported LLM provider: {provider}") + return provider_profile + + +def _legacy_connection( + profile: LLMProviderProfile, + *, + openai_api_key: str | None, + deepseek_api_key: str | None, + deepseek_base_url: str | None, + local_api_key: str | None, + local_base_url: str | None, + openai_compatible_api_key: str | None, + openai_compatible_base_url: str | None, + openrouter_api_key: str | None, + openrouter_base_url: str | None, + anthropic_api_key: str | None, + anthropic_base_url: str | None, + bedrock_api_key: str | None, + bedrock_base_url: str | None, + gemini_api_key: str | None, + gemini_base_url: str | None, +) -> LLMProviderConnection: + if profile.name == "openai": + return LLMProviderConnection(api_key=openai_api_key) + if profile.name == "deepseek": + return LLMProviderConnection(api_key=deepseek_api_key, base_url=deepseek_base_url) + if profile.name == "local": + return LLMProviderConnection(api_key=local_api_key, base_url=local_base_url) + if profile.name == "openai-compatible": + return LLMProviderConnection( + api_key=openai_compatible_api_key, + base_url=openai_compatible_base_url, + ) + if profile.name == "openrouter": + return LLMProviderConnection(api_key=openrouter_api_key, base_url=openrouter_base_url) + if profile.name == "anthropic": + return LLMProviderConnection(api_key=anthropic_api_key, base_url=anthropic_base_url) + if profile.name == "bedrock": + return LLMProviderConnection(api_key=bedrock_api_key, base_url=bedrock_base_url) + if profile.name == "gemini": + return LLMProviderConnection(api_key=gemini_api_key, base_url=gemini_base_url) + return profile.default_connection def _create_openai_client(settings: LLMClientSettings) -> LLMClient: return OpenAIResponsesClient( model=settings.model, - api_key=settings.openai_api_key, + api_key=settings.connection.api_key, timeout_seconds=settings.timeout_seconds, max_retries=settings.max_retries, ) @@ -107,8 +426,8 @@ def _create_openai_client(settings: LLMClientSettings) -> LLMClient: def _create_deepseek_client(settings: LLMClientSettings) -> LLMClient: return DeepSeekChatCompletionsClient( model=settings.model, - api_key=settings.deepseek_api_key, - base_url=settings.deepseek_base_url, + api_key=settings.connection.api_key, + base_url=settings.connection.base_url or DEFAULT_DEEPSEEK_BASE_URL, timeout_seconds=settings.timeout_seconds, max_retries=settings.max_retries, ) @@ -117,8 +436,58 @@ def _create_deepseek_client(settings: LLMClientSettings) -> LLMClient: def _create_local_client(settings: LLMClientSettings) -> LLMClient: return LocalOpenAICompatibleClient( model=settings.model, - api_key=settings.local_api_key, - base_url=settings.local_base_url, + api_key=settings.connection.api_key, + base_url=settings.connection.base_url or DEFAULT_LOCAL_BASE_URL, + timeout_seconds=settings.timeout_seconds, + max_retries=settings.max_retries, + ) + + +def _create_hosted_compatible_client(settings: LLMClientSettings) -> LLMClient: + return HostedOpenAICompatibleClient( + model=settings.model, + api_key=settings.connection.api_key, + base_url=settings.connection.base_url, + timeout_seconds=settings.timeout_seconds, + max_retries=settings.max_retries, + ) + + +def _create_openrouter_client(settings: LLMClientSettings) -> LLMClient: + return OpenRouterChatCompletionsClient( + model=settings.model, + api_key=settings.connection.api_key, + base_url=settings.connection.base_url or DEFAULT_OPENROUTER_BASE_URL, + timeout_seconds=settings.timeout_seconds, + max_retries=settings.max_retries, + ) + + +def _create_anthropic_client(settings: LLMClientSettings) -> LLMClient: + return AnthropicMessagesClient( + model=settings.model, + api_key=settings.connection.api_key, + base_url=settings.connection.base_url, + timeout_seconds=settings.timeout_seconds, + max_retries=settings.max_retries, + ) + + +def _create_bedrock_client(settings: LLMClientSettings) -> LLMClient: + return BedrockOpenAICompatibleClient( + model=settings.model, + api_key=settings.connection.api_key, + base_url=settings.connection.base_url, + timeout_seconds=settings.timeout_seconds, + max_retries=settings.max_retries, + ) + + +def _create_gemini_client(settings: LLMClientSettings) -> LLMClient: + return GeminiGenerateContentClient( + model=settings.model, + api_key=settings.connection.api_key, + base_url=settings.connection.base_url, timeout_seconds=settings.timeout_seconds, max_retries=settings.max_retries, ) diff --git a/chulk/llm/providers/__init__.py b/chulk/llm/providers/__init__.py index e5d264d..7228e41 100644 --- a/chulk/llm/providers/__init__.py +++ b/chulk/llm/providers/__init__.py @@ -1,7 +1,29 @@ """Hosted LLM provider implementations.""" +from chulk.llm.providers.anthropic import AnthropicMessagesClient +from chulk.llm.providers.bedrock import BedrockOpenAICompatibleClient +from chulk.llm.providers.chat_completions import ( + ChatCompletionsTransportProfile, + OpenAICompatibleChatCompletionsClient, +) from chulk.llm.providers.deepseek import DeepSeekChatCompletionsClient +from chulk.llm.providers.gemini import GeminiGenerateContentClient +from chulk.llm.providers.compatible import ( + HostedOpenAICompatibleClient, + OpenRouterChatCompletionsClient, +) from chulk.llm.providers.local import LocalOpenAICompatibleClient from chulk.llm.providers.openai import OpenAIResponsesClient -__all__ = ["DeepSeekChatCompletionsClient", "LocalOpenAICompatibleClient", "OpenAIResponsesClient"] +__all__ = [ + "AnthropicMessagesClient", + "BedrockOpenAICompatibleClient", + "ChatCompletionsTransportProfile", + "DeepSeekChatCompletionsClient", + "HostedOpenAICompatibleClient", + "GeminiGenerateContentClient", + "LocalOpenAICompatibleClient", + "OpenAICompatibleChatCompletionsClient", + "OpenRouterChatCompletionsClient", + "OpenAIResponsesClient", +] diff --git a/chulk/llm/providers/anthropic.py b/chulk/llm/providers/anthropic.py new file mode 100644 index 0000000..2ae1d3e --- /dev/null +++ b/chulk/llm/providers/anthropic.py @@ -0,0 +1,711 @@ +"""Anthropic Messages API provider client.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from typing import Any + +from chulk.llm.base import ( + LLMClient, + LLMConfigurationError, + LLMError, + LLMStreamChunk, + is_action_transport_fallback_error, + provider_error_from_exception, +) +from chulk.llm.capabilities import LLMCapabilities +from chulk.llm.messages import split_instructions +from chulk.llm.pricing import estimate_cost +from chulk.llm.tools import ( + action_payload_json, + native_final_answer_payload, + native_tool_action_payload, + parse_native_arguments, + provider_action_tools, + public_value, + with_json_action_prompt, +) +from chulk.llm.usage import LLMResponse, LLMUsage + + +DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS = 4096 + +ANTHROPIC_CAPABILITIES = LLMCapabilities( + supports_structured_output=False, + supports_json_mode=False, + supports_streaming=True, + supports_native_tool_calling=True, + supports_hosted_mcp_tools=False, + api_style="messages", +) + + +class AnthropicMessagesClient(LLMClient): + """LLM client backed by Anthropic's native Messages API.""" + + capabilities = ANTHROPIC_CAPABILITIES + provider = "anthropic" + + def __init__( + self, + *, + model: str, + api_key: str | None = None, + base_url: str | None = None, + timeout_seconds: float = 60.0, + max_retries: int = 2, + max_output_tokens: int = DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS, + client: Any | None = None, + async_client: Any | None = None, + ) -> None: + self.model = model.strip() + if not self.model: + raise LLMConfigurationError( + "CHULK_MODEL is required for the Anthropic LLM client", + provider=self.provider, + ) + self.max_output_tokens = _validate_max_output_tokens(max_output_tokens) + self._async_client = async_client + + if client is not None: + self._client = client + return + + resolved_api_key = api_key.strip() if api_key else "" + if not resolved_api_key: + raise LLMConfigurationError( + "ANTHROPIC_API_KEY is required for the Anthropic LLM client", + provider=self.provider, + model=self.model, + ) + + try: + from anthropic import Anthropic + except ImportError as exc: + raise LLMConfigurationError( + "The anthropic package is required. Install it with: pip install -e '.[anthropic]'", + provider=self.provider, + model=self.model, + ) from exc + + client_kwargs: dict[str, Any] = { + "api_key": resolved_api_key, + "timeout": timeout_seconds, + "max_retries": max_retries, + } + if base_url: + client_kwargs["base_url"] = base_url + self._client = Anthropic(**client_kwargs) + if async_client is not None: + self._async_client = async_client + else: + try: + from anthropic import AsyncAnthropic + except ImportError: + self._async_client = None + else: + self._async_client = AsyncAnthropic(**client_kwargs) + + def complete(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> str: + """Return a text response using Anthropic's Messages API.""" + return self.complete_response(messages, max_output_tokens=max_output_tokens).content + + def complete_response( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + """Return text plus normalized Anthropic usage metadata.""" + response = self._create( + self._request(messages, max_output_tokens=max_output_tokens), + operation="request", + ) + content = _response_text( + response, + provider=self.provider, + model=self.model, + action_transport=False, + ) + return self._response_from_provider(messages, content, _value(response, "usage")) + + async def acomplete_response( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + """Return text through Anthropic's native async Messages API.""" + if self._async_client is None: + return await super().acomplete_response(messages, max_output_tokens=max_output_tokens) + response = await self._acreate( + self._request(messages, max_output_tokens=max_output_tokens), + operation="request", + ) + content = _response_text( + response, + provider=self.provider, + model=self.model, + action_transport=False, + ) + return self._response_from_provider(messages, content, _value(response, "usage")) + + def stream_complete( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> Iterator[LLMStreamChunk]: + """Yield normalized text chunks from an Anthropic Messages stream.""" + request = self._request(messages, max_output_tokens=max_output_tokens) + request["stream"] = True + try: + stream = self._client.messages.create(**request) + except Exception as exc: + error = provider_error_from_exception( + exc, + message="Anthropic streaming request failed", + provider=self.provider, + model=self.model, + ) + if error is exc: + raise + raise error from exc + + text_parts: list[str] = [] + usage_parts: dict[str, int] = {} + stop_reason: object = None + try: + for event in stream: + event_type = _value(event, "type") + if event_type == "message_start": + _merge_stream_usage(usage_parts, _value(_value(event, "message"), "usage")) + continue + if event_type == "content_block_delta": + delta = _value(event, "delta") + if _value(delta, "type") != "text_delta": + continue + text = _value(delta, "text") + if isinstance(text, str) and text: + text_parts.append(text) + yield LLMStreamChunk( + type="text_delta", + text=text, + metadata={"event_type": event_type}, + ) + continue + if event_type == "message_delta": + _merge_stream_usage(usage_parts, _value(event, "usage")) + stop_reason = _value(_value(event, "delta"), "stop_reason") or stop_reason + continue + if event_type == "error": + raise LLMError( + f"Anthropic streaming request failed: {_event_error_message(event)}", + provider=self.provider, + model=self.model, + code="server_error", + retryable=True, + fallback_eligible=True, + ) + except Exception as exc: + error = provider_error_from_exception( + exc, + message="Anthropic streaming request failed", + provider=self.provider, + model=self.model, + ) + if error is exc: + raise + raise error from exc + + if not text_parts: + raise self._invalid_response_error("Anthropic streaming response did not include text content") + + content = "".join(text_parts) + response = self._response_from_provider(messages, content, usage_parts or None) + yield LLMStreamChunk( + type="completed", + metadata={"event_type": "message_stop", "stop_reason": stop_reason}, + usage=response.usage, + cost=response.cost, + ) + + def _complete_action_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> str: + return self._complete_action_response_once(messages, max_output_tokens=max_output_tokens).content + + def _complete_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, + ) -> LLMResponse: + del mcp_approval_callback + if hosted_mcp_servers: + raise LLMError( + "Anthropic Messages does not support Chulk hosted MCP tools", + provider=self.provider, + model=self.model, + code="unsupported_feature", + ) + if tools is not None: + try: + return self._complete_native_action_response_once( + messages, + tools=tools, + max_output_tokens=max_output_tokens, + ) + except LLMError as exc: + if not is_action_transport_fallback_error(exc): + raise + fallback = self._complete_json_action_response_once( + with_json_action_prompt(messages), + max_output_tokens=max_output_tokens, + ) + fallback.metadata.update( + { + "action_transport": "chulk_json_fallback", + "native_tool_call_error": str(exc), + } + ) + return fallback + return self._complete_json_action_response_once(messages, max_output_tokens=max_output_tokens) + + async def _acomplete_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, + ) -> LLMResponse: + del mcp_approval_callback + if self._async_client is None: + return await super()._acomplete_action_response_once( + messages, + max_output_tokens=max_output_tokens, + tools=tools, + hosted_mcp_servers=hosted_mcp_servers, + ) + if hosted_mcp_servers: + raise LLMError( + "Anthropic Messages does not support Chulk hosted MCP tools", + provider=self.provider, + model=self.model, + code="unsupported_feature", + ) + if tools is not None: + try: + return await self._acomplete_native_action_response_once( + messages, + tools=tools, + max_output_tokens=max_output_tokens, + ) + except LLMError as exc: + if not is_action_transport_fallback_error(exc): + raise + fallback = await self._acomplete_json_action_response_once( + with_json_action_prompt(messages), + max_output_tokens=max_output_tokens, + ) + fallback.metadata.update( + { + "action_transport": "chulk_json_fallback", + "native_tool_call_error": str(exc), + } + ) + return fallback + return await self._acomplete_json_action_response_once( + messages, + max_output_tokens=max_output_tokens, + ) + + def _complete_json_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + response = self._create( + self._request(messages, max_output_tokens=max_output_tokens), + operation="structured action request", + ) + content = _response_text( + response, + provider=self.provider, + model=self.model, + action_transport=True, + ) + result = self._response_from_provider(messages, content, _value(response, "usage")) + result.metadata.update({"action_transport": "chulk_json"}) + return result + + async def _acomplete_json_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + response = await self._acreate( + self._request(messages, max_output_tokens=max_output_tokens), + operation="structured action request", + ) + content = _response_text( + response, + provider=self.provider, + model=self.model, + action_transport=True, + ) + result = self._response_from_provider(messages, content, _value(response, "usage")) + result.metadata.update({"action_transport": "chulk_json"}) + return result + + def _complete_native_action_response_once( + self, + messages: list[dict[str, str]], + *, + tools: list[object], + max_output_tokens: int | None = None, + ) -> LLMResponse: + request = self._request(messages, max_output_tokens=max_output_tokens) + request.update( + { + "tools": _anthropic_tools(tools), + "tool_choice": {"type": "auto", "disable_parallel_tool_use": True}, + } + ) + response = self._create( + request, + operation="native tool action request", + action_transport=True, + ) + content, raw_tool_call = _normalize_native_action_response( + response, + provider=self.provider, + model=self.model, + ) + result = self._response_from_provider(messages, content, _value(response, "usage")) + result.metadata.update( + { + "action_transport": "provider_native", + "provider_tool_call": raw_tool_call, + } + ) + return result + + async def _acomplete_native_action_response_once( + self, + messages: list[dict[str, str]], + *, + tools: list[object], + max_output_tokens: int | None = None, + ) -> LLMResponse: + request = self._request(messages, max_output_tokens=max_output_tokens) + request.update( + { + "tools": _anthropic_tools(tools), + "tool_choice": {"type": "auto", "disable_parallel_tool_use": True}, + } + ) + response = await self._acreate( + request, + operation="native tool action request", + action_transport=True, + ) + content, raw_tool_call = _normalize_native_action_response( + response, + provider=self.provider, + model=self.model, + ) + result = self._response_from_provider(messages, content, _value(response, "usage")) + result.metadata.update( + { + "action_transport": "provider_native", + "provider_tool_call": raw_tool_call, + } + ) + return result + + def _request( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None, + ) -> dict[str, Any]: + system, conversation = split_instructions(messages) + request: dict[str, Any] = { + "model": self.model, + "messages": _normalize_conversation(conversation), + "max_tokens": ( + self.max_output_tokens + if max_output_tokens is None + else _validate_max_output_tokens(max_output_tokens) + ), + } + if system: + request["system"] = system + return request + + def _create( + self, + request: dict[str, Any], + *, + operation: str, + action_transport: bool = False, + ) -> object: + try: + return self._client.messages.create(**request) + except Exception as exc: + error = provider_error_from_exception( + exc, + message=f"Anthropic {operation} failed", + provider=self.provider, + model=self.model, + action_transport=action_transport, + ) + if error is exc: + raise + raise error from exc + + async def _acreate( + self, + request: dict[str, Any], + *, + operation: str, + action_transport: bool = False, + ) -> object: + async_client = self._async_client + if async_client is None: + raise RuntimeError("Async Anthropic client is not configured") + try: + return await async_client.messages.create(**request) + except Exception as exc: + error = provider_error_from_exception( + exc, + message=f"Anthropic {operation} failed", + provider=self.provider, + model=self.model, + action_transport=action_transport, + ) + if error is exc: + raise + raise error from exc + + def _response_from_provider( + self, + messages: list[dict[str, str]], + content: str, + usage_payload: object, + ) -> LLMResponse: + usage = normalize_anthropic_usage(usage_payload) + if usage is None: + return self._response_with_estimated_usage(messages, content) + return LLMResponse( + content=content, + usage=usage, + cost=estimate_cost(self.provider, self.model, usage), + provider=self.provider, + model=self.model, + ) + + def _invalid_response_error(self, message: str) -> LLMError: + return LLMError( + message, + provider=self.provider, + model=self.model, + code="invalid_response", + retryable=True, + fallback_eligible=True, + ) + + +def normalize_anthropic_usage(usage: object) -> LLMUsage | None: + """Return normalized usage from an Anthropic Messages response.""" + if usage is None: + return None + uncached_input = _int_value(_value(usage, "input_tokens")) + output_tokens = _int_value(_value(usage, "output_tokens")) + cache_creation = _int_value(_value(usage, "cache_creation_input_tokens")) + cache_read = _int_value(_value(usage, "cache_read_input_tokens")) + input_tokens = uncached_input + cache_creation + cache_read + if not any([input_tokens, output_tokens]): + return None + return LLMUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, + cached_input_tokens=cache_read, + cache_hit_input_tokens=cache_read, + cache_miss_input_tokens=uncached_input + cache_creation, + estimated=False, + cache_split_estimated=False, + source="provider", + raw=_public_dict(usage), + ) + + +def _anthropic_tools(tools: list[object]) -> list[dict[str, Any]]: + return [ + { + "name": declaration["name"], + "description": declaration["description"], + "input_schema": declaration["parameters"], + } + for declaration in provider_action_tools(tools) + ] + + +def _normalize_native_action_response( + response: object, + *, + provider: str, + model: str, +) -> tuple[str, dict[str, Any] | None]: + content = _value(response, "content") + if not isinstance(content, list): + raise LLMError( + "Anthropic native action response did not include content blocks", + provider=provider, + model=model, + code="action_shape_error", + ) + + tool_blocks = [block for block in content if _value(block, "type") == "tool_use"] + if len(tool_blocks) > 1: + raise LLMError( + "Anthropic native action response included multiple tool calls", + provider=provider, + model=model, + code="action_shape_error", + ) + if tool_blocks: + block = tool_blocks[0] + name = _value(block, "name") + if not isinstance(name, str) or not name: + raise LLMError( + "Anthropic native tool call did not include a function name", + provider=provider, + model=model, + code="action_shape_error", + ) + try: + arguments = parse_native_arguments(_value(block, "input")) + except ValueError as exc: + raise LLMError( + str(exc), + provider=provider, + model=model, + code="action_shape_error", + ) from exc + return action_payload_json(native_tool_action_payload(name, arguments)), public_value(block) + + text = _text_from_content(content) + if text: + return action_payload_json(native_final_answer_payload(text)), None + raise LLMError( + "Anthropic native action response did not include a tool call or text content", + provider=provider, + model=model, + code="action_shape_error", + ) + + +def _response_text( + response: object, + *, + provider: str, + model: str, + action_transport: bool, +) -> str: + content = _value(response, "content") + text = _text_from_content(content) + if text: + return text + raise LLMError( + "Anthropic response did not include text content", + provider=provider, + model=model, + code="action_shape_error" if action_transport else "invalid_response", + retryable=not action_transport, + fallback_eligible=True, + ) + + +def _text_from_content(content: object) -> str: + if not isinstance(content, list): + return "" + parts = [ + text + for block in content + if _value(block, "type") == "text" + for text in [_value(block, "text")] + if isinstance(text, str) and text + ] + return "".join(parts).strip() + + +def _normalize_conversation(messages: list[dict[str, str]]) -> list[dict[str, str]]: + normalized: list[dict[str, str]] = [] + for message in messages: + role = message.get("role") + if role not in {"user", "assistant"}: + role = "user" + content = message.get("content", "") + if normalized and normalized[-1]["role"] == role: + normalized[-1]["content"] = "\n\n".join([normalized[-1]["content"], content]) + else: + normalized.append({"role": role, "content": content}) + if not normalized: + normalized.append({"role": "user", "content": "Continue from the available context."}) + return normalized + + +def _merge_stream_usage(target: dict[str, int], usage: object) -> None: + if usage is None: + return + for key in ( + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + ): + value = _int_value(_value(usage, key)) + if value or key not in target: + target[key] = value + + +def _event_error_message(event: object) -> str: + error = _value(event, "error") + message = _value(error, "message") + if isinstance(message, str) and message: + return message + return str(error or event) + + +def _validate_max_output_tokens(value: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError("max_output_tokens must be greater than zero") + return value + + +def _value(value: object, key: str) -> object: + if isinstance(value, dict): + return value.get(key) + return getattr(value, key, None) + + +def _int_value(value: object) -> int: + return value if isinstance(value, int) and not isinstance(value, bool) else 0 + + +def _public_dict(value: object) -> dict[str, Any]: + public = public_value(value) + return public if isinstance(public, dict) else {} diff --git a/chulk/llm/providers/bedrock.py b/chulk/llm/providers/bedrock.py new file mode 100644 index 0000000..bf950c9 --- /dev/null +++ b/chulk/llm/providers/bedrock.py @@ -0,0 +1,89 @@ +"""AWS Bedrock client for its OpenAI-compatible Chat Completions API.""" + +from __future__ import annotations + +from typing import Any + +from chulk.llm.base import LLMConfigurationError +from chulk.llm.capabilities import LLMCapabilities +from chulk.llm.messages import chat_messages +from chulk.llm.providers.chat_completions import ( + ChatCompletionsTransportProfile, + OpenAICompatibleChatCompletionsClient, +) +from chulk.llm.usage import normalize_chat_completions_usage + + +BEDROCK_CAPABILITIES = LLMCapabilities( + supports_structured_output=False, + supports_json_mode=False, + supports_streaming=True, + supports_native_tool_calling=True, + api_style="chat_completions", +) + +BEDROCK_TRANSPORT_PROFILE = ChatCompletionsTransportProfile( + provider="bedrock", + display_name="AWS Bedrock", + capabilities=BEDROCK_CAPABILITIES, + normalize_messages=chat_messages, + normalize_usage=normalize_chat_completions_usage, + missing_api_key_message=( + "BEDROCK_API_KEY or AWS_BEARER_TOKEN_BEDROCK is required for AWS Bedrock" + ), +) + + +class BedrockOpenAICompatibleClient(OpenAICompatibleChatCompletionsClient): + """Client for a caller-selected Bedrock OpenAI-compatible endpoint.""" + + capabilities = BEDROCK_CAPABILITIES + provider = "bedrock" + + def __init__( + self, + *, + model: str | None = None, + api_key: str | None = None, + base_url: str | None = None, + timeout_seconds: float = 60.0, + max_retries: int = 2, + client: Any | None = None, + async_client: Any | None = None, + ) -> None: + normalized_model = _required_value(model, "model") + normalized_api_key = _required_value( + api_key, + "api_key", + model=normalized_model, + ) + normalized_base_url = _required_value( + base_url, + "base_url", + model=normalized_model, + ) + super().__init__( + profile=BEDROCK_TRANSPORT_PROFILE, + model=normalized_model, + api_key=normalized_api_key, + base_url=normalized_base_url, + timeout_seconds=timeout_seconds, + max_retries=max_retries, + client=client, + async_client=async_client, + ) + + +def _required_value( + value: str | None, + field: str, + *, + model: str | None = None, +) -> str: + if value is None or not value.strip(): + raise LLMConfigurationError( + f"{field} is required for provider bedrock", + provider="bedrock", + model=model, + ) + return value.strip() diff --git a/chulk/llm/providers/chat_completions.py b/chulk/llm/providers/chat_completions.py new file mode 100644 index 0000000..af610dd --- /dev/null +++ b/chulk/llm/providers/chat_completions.py @@ -0,0 +1,600 @@ +"""Shared transport for OpenAI-compatible Chat Completions providers.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from typing import Any + +from chulk.llm.base import ( + LLMClient, + LLMConfigurationError, + LLMError, + LLMStreamChunk, + is_action_transport_fallback_error, + provider_error_from_exception, +) +from chulk.llm.capabilities import LLMCapabilities +from chulk.llm.pricing import estimate_cost +from chulk.llm.tools import ( + action_payload_json, + chat_completion_tools, + native_final_answer_payload, + native_tool_action_payload, + parse_native_arguments, + public_value, + with_json_action_prompt, +) +from chulk.llm.usage import LLMResponse, LLMUsage + + +MessageNormalizer = Callable[[list[dict[str, str]]], list[dict[str, str]]] +UsageNormalizer = Callable[[object], LLMUsage | None] + + +@dataclass(frozen=True) +class ChatCompletionsTransportProfile: + """Provider hooks used by the shared Chat Completions transport.""" + + provider: str + display_name: str + capabilities: LLMCapabilities + normalize_messages: MessageNormalizer + normalize_usage: UsageNormalizer + json_response_format: dict[str, Any] | None = None + missing_api_key_message: str | None = None + default_api_key: str | None = None + + +class OpenAICompatibleChatCompletionsClient(LLMClient): + """Explicit provider-neutral implementation of Chat Completions plumbing.""" + + def __init__( + self, + *, + profile: ChatCompletionsTransportProfile, + model: str, + api_key: str | None, + base_url: str, + timeout_seconds: float, + max_retries: int, + client: Any | None, + async_client: Any | None = None, + ) -> None: + self.profile = profile + self.provider = profile.provider + self.capabilities = profile.capabilities + self.model = model + self.base_url = _validate_base_url(base_url) + self._async_client = async_client + + if client is not None: + self._client = client + return + + resolved_api_key = api_key or profile.default_api_key + if not resolved_api_key and profile.missing_api_key_message: + raise LLMConfigurationError( + profile.missing_api_key_message, + provider=self.provider, + model=self.model, + ) + + try: + from openai import OpenAI + except ImportError as exc: + raise LLMConfigurationError( + "The openai package is required. Install it with: pip install -e '.[openai]'", + provider=self.provider, + model=self.model, + ) from exc + + try: + from openai import AsyncOpenAI + except ImportError: + AsyncOpenAI = None # type: ignore[misc, assignment] + + self._client = OpenAI( + api_key=resolved_api_key, + base_url=self.base_url, + timeout=timeout_seconds, + max_retries=max_retries, + ) + if async_client is not None: + self._async_client = async_client + elif AsyncOpenAI is not None: + self._async_client = AsyncOpenAI( + api_key=resolved_api_key, + base_url=self.base_url, + timeout=timeout_seconds, + max_retries=max_retries, + ) + + def complete(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> str: + """Return a text response through the configured Chat Completions endpoint.""" + return self.complete_response(messages, max_output_tokens=max_output_tokens).content + + def complete_response( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + """Return a text response plus normalized provider usage.""" + request = self._request(messages, max_output_tokens=max_output_tokens) + response = self._create(request, operation="request") + content = self._message_content(response, operation="response") + return self._response_from_provider(messages, content, getattr(response, "usage", None)) + + async def acomplete_response( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + """Return text through the native async Chat Completions transport.""" + if self._async_client is None: + return await super().acomplete_response(messages, max_output_tokens=max_output_tokens) + request = self._request(messages, max_output_tokens=max_output_tokens) + response = await self._acreate(request, operation="request") + content = self._message_content(response, operation="response") + return self._response_from_provider(messages, content, getattr(response, "usage", None)) + + def stream_complete( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> Iterator[LLMStreamChunk]: + """Yield normalized chunks from a Chat Completions stream.""" + request = self._request(messages, max_output_tokens=max_output_tokens, stream=True) + try: + stream = self._client.chat.completions.create(**request) + except Exception as exc: + error = provider_error_from_exception( + exc, + message=f"{self.profile.display_name} streaming request failed", + provider=self.provider, + model=self.model, + ) + if error is exc: + raise + raise error from exc + + text_parts: list[str] = [] + usage_payload: object = None + finish_reason: object = None + try: + for chunk in stream: + chunk_usage = _value(chunk, "usage") + if chunk_usage is not None: + usage_payload = chunk_usage + choice = _first_choice(chunk) + if choice is None: + continue + finish_reason = _value(choice, "finish_reason") or finish_reason + delta = _value(_value(choice, "delta"), "content") + if isinstance(delta, str) and delta: + text_parts.append(delta) + yield LLMStreamChunk( + type="text_delta", + text=delta, + metadata={"transport": "chat_completions"}, + ) + except Exception as exc: + error = provider_error_from_exception( + exc, + message=f"{self.profile.display_name} streaming request failed", + provider=self.provider, + model=self.model, + ) + if error is exc: + raise + raise error from exc + + if not text_parts: + raise LLMError( + f"{self.profile.display_name} streaming response did not include message content", + provider=self.provider, + model=self.model, + code="invalid_response", + retryable=True, + fallback_eligible=True, + ) + + content = "".join(text_parts) + response = self._response_from_provider(messages, content, usage_payload) + yield LLMStreamChunk( + type="completed", + metadata={ + "transport": "chat_completions", + "finish_reason": finish_reason, + }, + usage=response.usage, + cost=response.cost, + ) + + def _complete_action_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> str: + return self._complete_action_response_once(messages, max_output_tokens=max_output_tokens).content + + def _complete_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, + ) -> LLMResponse: + if tools is not None: + try: + return self._complete_native_action_response_once( + messages, + tools=tools, + max_output_tokens=max_output_tokens, + ) + except LLMError as exc: + if not is_action_transport_fallback_error(exc): + raise + fallback = self._complete_json_action_response_once( + with_json_action_prompt(messages), + max_output_tokens=max_output_tokens, + ) + fallback.metadata.update( + { + "action_transport": "chulk_json_fallback", + "native_tool_call_error": str(exc), + } + ) + return fallback + return self._complete_json_action_response_once(messages, max_output_tokens=max_output_tokens) + + async def _acomplete_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, + ) -> LLMResponse: + if self._async_client is None: + return await super()._acomplete_action_response_once( + messages, + max_output_tokens=max_output_tokens, + tools=tools, + hosted_mcp_servers=hosted_mcp_servers, + mcp_approval_callback=mcp_approval_callback, + ) + if tools is not None: + try: + return await self._acomplete_native_action_response_once( + messages, + tools=tools, + max_output_tokens=max_output_tokens, + ) + except LLMError as exc: + if not is_action_transport_fallback_error(exc): + raise + fallback = await self._acomplete_json_action_response_once( + with_json_action_prompt(messages), + max_output_tokens=max_output_tokens, + ) + fallback.metadata.update( + { + "action_transport": "chulk_json_fallback", + "native_tool_call_error": str(exc), + } + ) + return fallback + return await self._acomplete_json_action_response_once( + messages, + max_output_tokens=max_output_tokens, + ) + + def _complete_json_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + request = self._request(messages, max_output_tokens=max_output_tokens) + if self.profile.json_response_format is not None: + request["response_format"] = dict(self.profile.json_response_format) + response = self._create(request, operation="structured action request") + content = self._message_content(response, operation="structured action response") + result = self._response_from_provider(messages, content, getattr(response, "usage", None)) + result.metadata.update({"action_transport": "chulk_json"}) + return result + + async def _acomplete_json_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + request = self._request(messages, max_output_tokens=max_output_tokens) + if self.profile.json_response_format is not None: + request["response_format"] = dict(self.profile.json_response_format) + response = await self._acreate(request, operation="structured action request") + content = self._message_content(response, operation="structured action response") + result = self._response_from_provider(messages, content, getattr(response, "usage", None)) + result.metadata.update({"action_transport": "chulk_json"}) + return result + + def _complete_native_action_response_once( + self, + messages: list[dict[str, str]], + *, + tools: list[object], + max_output_tokens: int | None = None, + ) -> LLMResponse: + request = self._request(messages, max_output_tokens=max_output_tokens) + request.update( + { + "tools": chat_completion_tools(tools), + "tool_choice": "auto", + } + ) + response = self._create(request, operation="native tool action request", action_transport=True) + message = _response_message( + response, + display_name=self.profile.display_name, + provider=self.provider, + model=self.model, + action_transport=True, + ) + content, raw_tool_call = _normalize_native_action_message( + message, + provider=self.provider, + model=self.model, + ) + result = self._response_from_provider(messages, content, getattr(response, "usage", None)) + result.metadata.update( + { + "action_transport": "provider_native", + "provider_tool_call": raw_tool_call, + } + ) + return result + + async def _acomplete_native_action_response_once( + self, + messages: list[dict[str, str]], + *, + tools: list[object], + max_output_tokens: int | None = None, + ) -> LLMResponse: + request = self._request(messages, max_output_tokens=max_output_tokens) + request.update( + { + "tools": chat_completion_tools(tools), + "tool_choice": "auto", + } + ) + response = await self._acreate( + request, + operation="native tool action request", + action_transport=True, + ) + message = _response_message( + response, + display_name=self.profile.display_name, + provider=self.provider, + model=self.model, + action_transport=True, + ) + content, raw_tool_call = _normalize_native_action_message( + message, + provider=self.provider, + model=self.model, + ) + result = self._response_from_provider(messages, content, getattr(response, "usage", None)) + result.metadata.update( + { + "action_transport": "provider_native", + "provider_tool_call": raw_tool_call, + } + ) + return result + + def _request( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None, + stream: bool = False, + ) -> dict[str, Any]: + request: dict[str, Any] = { + "model": self.model, + "messages": self.profile.normalize_messages(messages), + "stream": stream, + } + output_limit = _validate_max_output_tokens(max_output_tokens) + if output_limit is not None: + request["max_tokens"] = output_limit + return request + + def _create( + self, + request: dict[str, Any], + *, + operation: str, + action_transport: bool = False, + ) -> object: + try: + return self._client.chat.completions.create(**request) + except Exception as exc: + error = provider_error_from_exception( + exc, + message=f"{self.profile.display_name} {operation} failed", + provider=self.provider, + model=self.model, + action_transport=action_transport, + ) + if error is exc: + raise + raise error from exc + + async def _acreate( + self, + request: dict[str, Any], + *, + operation: str, + action_transport: bool = False, + ) -> object: + async_client = self._async_client + if async_client is None: + raise RuntimeError("Async Chat Completions client is not configured") + try: + return await async_client.chat.completions.create(**request) + except Exception as exc: + error = provider_error_from_exception( + exc, + message=f"{self.profile.display_name} {operation} failed", + provider=self.provider, + model=self.model, + action_transport=action_transport, + ) + if error is exc: + raise + raise error from exc + + def _message_content(self, response: object, *, operation: str) -> str: + message = _response_message( + response, + display_name=self.profile.display_name, + provider=self.provider, + model=self.model, + ) + content = _value(message, "content") + if isinstance(content, str) and content: + return content + raise LLMError( + f"{self.profile.display_name} {operation} content was empty", + provider=self.provider, + model=self.model, + code="invalid_response", + retryable=True, + fallback_eligible=True, + ) + + def _response_from_provider( + self, + messages: list[dict[str, str]], + content: str, + usage_payload: object, + ) -> LLMResponse: + usage = self.profile.normalize_usage(usage_payload) + if usage is None: + return self._response_with_estimated_usage(messages, content) + return LLMResponse( + content=content, + usage=usage, + cost=estimate_cost(self.profile.provider, self.model, usage), + provider=self.profile.provider, + model=self.model, + ) + + +def _validate_base_url(value: str) -> str: + if not value.strip(): + raise ValueError("base_url must be non-empty") + return value.strip() + + +def _validate_max_output_tokens(value: int | None) -> int | None: + if value is None: + return None + if value < 1: + raise ValueError("max_output_tokens must be greater than zero") + return value + + +def _response_message( + response: object, + *, + display_name: str, + provider: str, + model: str, + action_transport: bool = False, +) -> object: + choice = _first_choice(response) + message = _value(choice, "message") if choice is not None else None + if message is None: + raise LLMError( + f"{display_name} response did not include message content", + provider=provider, + model=model, + code="action_shape_error" if action_transport else "invalid_response", + retryable=not action_transport, + fallback_eligible=not action_transport, + ) + return message + + +def _first_choice(response: object) -> object | None: + choices = _value(response, "choices") + if isinstance(choices, (list, tuple)) and choices: + return choices[0] + return None + + +def _normalize_native_action_message( + message: object, + *, + provider: str, + model: str, +) -> tuple[str, dict[str, Any] | None]: + tool_calls = _value(message, "tool_calls") + if isinstance(tool_calls, (list, tuple)) and tool_calls: + if len(tool_calls) != 1: + raise LLMError( + "Native action response included multiple tool calls; Chulk accepts one action per turn", + provider=provider, + model=model, + code="action_shape_error", + ) + tool_call = tool_calls[0] + function = _value(tool_call, "function") + name = _value(function, "name") + if not isinstance(name, str) or not name: + raise LLMError( + "Native tool call did not include a function name", + provider=provider, + model=model, + code="action_shape_error", + ) + try: + arguments = parse_native_arguments(_value(function, "arguments")) + except ValueError as exc: + raise LLMError( + str(exc), + provider=provider, + model=model, + code="action_shape_error", + ) from exc + payload = native_tool_action_payload(name, arguments) + raw_tool_call = public_value(tool_call) + return action_payload_json(payload), raw_tool_call if isinstance(raw_tool_call, dict) else None + + content = _value(message, "content") + if isinstance(content, str) and content.strip(): + return action_payload_json(native_final_answer_payload(content.strip())), None + raise LLMError( + "Native action response did not include a tool call or content", + provider=provider, + model=model, + code="action_shape_error", + ) + + +def _value(source: object, key: str) -> object: + if isinstance(source, dict): + return source.get(key) + return getattr(source, key, None) diff --git a/chulk/llm/providers/compatible.py b/chulk/llm/providers/compatible.py new file mode 100644 index 0000000..bff8311 --- /dev/null +++ b/chulk/llm/providers/compatible.py @@ -0,0 +1,243 @@ +"""Hosted OpenAI-compatible and OpenRouter provider clients.""" + +from __future__ import annotations + +from typing import Any + +from chulk.llm.base import LLMConfigurationError +from chulk.llm.capabilities import LLMCapabilities +from chulk.llm.messages import chat_messages +from chulk.llm.providers.chat_completions import ( + ChatCompletionsTransportProfile, + OpenAICompatibleChatCompletionsClient, +) +from chulk.llm.usage import normalize_chat_completions_usage + + +DEFAULT_OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1" + +HOSTED_OPENAI_COMPATIBLE_CAPABILITIES = LLMCapabilities( + supports_structured_output=False, + supports_json_mode=False, + supports_streaming=True, + supports_native_tool_calling=True, + api_style="chat_completions", +) + +OPENROUTER_CAPABILITIES = LLMCapabilities( + supports_structured_output=False, + supports_json_mode=False, + supports_streaming=True, + supports_native_tool_calling=True, + api_style="chat_completions", +) + +HOSTED_OPENAI_COMPATIBLE_TRANSPORT_PROFILE = ChatCompletionsTransportProfile( + provider="openai-compatible", + display_name="Hosted OpenAI-compatible provider", + capabilities=HOSTED_OPENAI_COMPATIBLE_CAPABILITIES, + normalize_messages=chat_messages, + normalize_usage=normalize_chat_completions_usage, + missing_api_key_message=( + "CHULK_OPENAI_COMPATIBLE_API_KEY is required for the hosted OpenAI-compatible provider" + ), +) + +OPENROUTER_TRANSPORT_PROFILE = ChatCompletionsTransportProfile( + provider="openrouter", + display_name="OpenRouter", + capabilities=OPENROUTER_CAPABILITIES, + normalize_messages=chat_messages, + normalize_usage=normalize_chat_completions_usage, + missing_api_key_message="OPENROUTER_API_KEY or CHULK_OPENROUTER_API_KEY is required for OpenRouter", +) + + +class HostedOpenAICompatibleClient(OpenAICompatibleChatCompletionsClient): + """Client for a user-selected hosted Chat Completions endpoint.""" + + capabilities = HOSTED_OPENAI_COMPATIBLE_CAPABILITIES + provider = "openai-compatible" + + def __init__( + self, + *, + model: str | None = None, + api_key: str | None = None, + base_url: str | None = None, + timeout_seconds: float = 60.0, + max_retries: int = 2, + client: Any | None = None, + async_client: Any | None = None, + ) -> None: + normalized_model = _required_value(model, "model", provider=self.provider) + normalized_api_key = _required_value( + api_key, + "api_key", + provider=self.provider, + model=normalized_model, + ) + normalized_base_url = _required_value( + base_url, + "base_url", + provider=self.provider, + model=normalized_model, + ) + super().__init__( + profile=HOSTED_OPENAI_COMPATIBLE_TRANSPORT_PROFILE, + model=normalized_model, + api_key=normalized_api_key, + base_url=normalized_base_url, + timeout_seconds=timeout_seconds, + max_retries=max_retries, + client=client, + async_client=async_client, + ) + + +class OpenRouterChatCompletionsClient(OpenAICompatibleChatCompletionsClient): + """Client for OpenRouter's OpenAI-compatible Chat Completions API.""" + + capabilities = OPENROUTER_CAPABILITIES + provider = "openrouter" + + def __init__( + self, + *, + model: str | None = None, + api_key: str | None = None, + base_url: str = DEFAULT_OPENROUTER_BASE_URL, + site_url: str | None = None, + app_name: str | None = None, + timeout_seconds: float = 60.0, + max_retries: int = 2, + client: Any | None = None, + async_client: Any | None = None, + ) -> None: + normalized_model = _required_value(model, "model", provider=self.provider) + normalized_api_key = _required_value( + api_key, + "api_key", + provider=self.provider, + model=normalized_model, + ) + normalized_base_url = _required_value( + base_url, + "base_url", + provider=self.provider, + model=normalized_model, + ) + self.default_headers = openrouter_default_headers( + site_url=site_url, app_name=app_name + ) + build_sync_client = client is None + if build_sync_client and self.default_headers: + client = _openai_sdk_client( + api_key=normalized_api_key, + base_url=normalized_base_url, + timeout_seconds=timeout_seconds, + max_retries=max_retries, + default_headers=self.default_headers, + model=normalized_model, + ) + if async_client is None and build_sync_client and self.default_headers: + async_client = _openai_sdk_async_client( + api_key=normalized_api_key, + base_url=normalized_base_url, + timeout_seconds=timeout_seconds, + max_retries=max_retries, + default_headers=self.default_headers, + model=normalized_model, + ) + super().__init__( + profile=OPENROUTER_TRANSPORT_PROFILE, + model=normalized_model, + api_key=normalized_api_key, + base_url=normalized_base_url, + timeout_seconds=timeout_seconds, + max_retries=max_retries, + client=client, + async_client=async_client, + ) + + +def openrouter_default_headers( + *, + site_url: str | None = None, + app_name: str | None = None, +) -> dict[str, str]: + """Return the optional OpenRouter attribution headers.""" + headers: dict[str, str] = {} + if site_url is not None and site_url.strip(): + headers["HTTP-Referer"] = site_url.strip() + if app_name is not None and app_name.strip(): + headers["X-OpenRouter-Title"] = app_name.strip() + return headers + + +def _required_value( + value: str | None, + field: str, + *, + provider: str, + model: str | None = None, +) -> str: + if value is None or not value.strip(): + raise LLMConfigurationError( + f"{field} is required for provider {provider}", + provider=provider, + model=model, + ) + return value.strip() + + +def _openai_sdk_client( + *, + api_key: str, + base_url: str, + timeout_seconds: float, + max_retries: int, + default_headers: dict[str, str], + model: str, +) -> Any: + try: + from openai import OpenAI + except ImportError as exc: + raise LLMConfigurationError( + "The openai package is required. Install it with: pip install -e '.[openai]'", + provider="openrouter", + model=model, + ) from exc + return OpenAI( + api_key=api_key, + base_url=base_url, + timeout=timeout_seconds, + max_retries=max_retries, + default_headers=default_headers, + ) + + +def _openai_sdk_async_client( + *, + api_key: str, + base_url: str, + timeout_seconds: float, + max_retries: int, + default_headers: dict[str, str], + model: str, +) -> Any: + try: + from openai import AsyncOpenAI + except ImportError as exc: + raise LLMConfigurationError( + "The openai package is required. Install it with: pip install -e '.[openai]'", + provider="openrouter", + model=model, + ) from exc + return AsyncOpenAI( + api_key=api_key, + base_url=base_url, + timeout=timeout_seconds, + max_retries=max_retries, + default_headers=default_headers, + ) diff --git a/chulk/llm/providers/deepseek.py b/chulk/llm/providers/deepseek.py index fb10728..294d169 100644 --- a/chulk/llm/providers/deepseek.py +++ b/chulk/llm/providers/deepseek.py @@ -4,32 +4,37 @@ from typing import Any -from chulk.llm.base import LLMClient, LLMConfigurationError, LLMError from chulk.llm.capabilities import LLMCapabilities from chulk.llm.messages import chat_messages -from chulk.llm.pricing import estimate_cost -from chulk.llm.tools import ( - action_payload_json, - chat_completion_tools, - native_final_answer_payload, - native_tool_action_payload, - parse_native_arguments, - public_value, - with_json_action_prompt, +from chulk.llm.providers.chat_completions import ( + ChatCompletionsTransportProfile, + OpenAICompatibleChatCompletionsClient, ) -from chulk.llm.usage import LLMResponse, normalize_deepseek_usage +from chulk.llm.usage import normalize_deepseek_usage +DEFAULT_DEEPSEEK_BASE_URL = "https://api.deepseek.com" + DEEPSEEK_CAPABILITIES = LLMCapabilities( supports_structured_output=False, supports_json_mode=True, - supports_streaming=False, + supports_streaming=True, supports_native_tool_calling=True, api_style="chat_completions", ) +DEEPSEEK_TRANSPORT_PROFILE = ChatCompletionsTransportProfile( + provider="deepseek", + display_name="DeepSeek", + capabilities=DEEPSEEK_CAPABILITIES, + normalize_messages=chat_messages, + normalize_usage=normalize_deepseek_usage, + json_response_format={"type": "json_object"}, + missing_api_key_message="DEEPSEEK_API_KEY or CHULK_DEEPSEEK_API_KEY is required for DeepSeek", +) + -class DeepSeekChatCompletionsClient(LLMClient): +class DeepSeekChatCompletionsClient(OpenAICompatibleChatCompletionsClient): """LLM client backed by DeepSeek's OpenAI-compatible Chat Completions API.""" capabilities = DEEPSEEK_CAPABILITIES @@ -40,179 +45,19 @@ def __init__( *, model: str, api_key: str | None = None, - base_url: str = "https://api.deepseek.com", + base_url: str = DEFAULT_DEEPSEEK_BASE_URL, timeout_seconds: float = 60.0, max_retries: int = 2, client: Any | None = None, + async_client: Any | None = None, ) -> None: - self.model = model - self.base_url = base_url - - if client is not None: - self._client = client - return - - if not api_key: - raise LLMConfigurationError("DEEPSEEK_API_KEY or CHULK_DEEPSEEK_API_KEY is required for DeepSeek") - - try: - from openai import OpenAI - except ImportError as exc: - raise LLMConfigurationError( - "The openai package is required. Install it with: pip install -e '.[openai]'" - ) from exc - - self._client = OpenAI( + super().__init__( + profile=DEEPSEEK_TRANSPORT_PROFILE, + model=model, api_key=api_key, base_url=base_url, - timeout=timeout_seconds, + timeout_seconds=timeout_seconds, max_retries=max_retries, + client=client, + async_client=async_client, ) - - def complete(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> str: - """Return a text response using DeepSeek chat completions.""" - return self.complete_response(messages, max_output_tokens=max_output_tokens).content - - def complete_response( - self, - messages: list[dict[str, str]], - *, - max_output_tokens: int | None = None, - ) -> LLMResponse: - """Return a text response plus DeepSeek usage metadata.""" - request = { - "model": self.model, - "messages": chat_messages(messages), - "stream": False, - } - try: - response = self._client.chat.completions.create(**request) - except Exception as exc: - raise LLMError(f"DeepSeek request failed: {exc}") from exc - - try: - content = response.choices[0].message.content - except (AttributeError, IndexError) as exc: - raise LLMError("DeepSeek response did not include message content") from exc - - if isinstance(content, str) and content: - return self._response_from_provider(messages, content, getattr(response, "usage", None)) - raise LLMError("DeepSeek response content was empty") - - def _complete_action_once(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> str: - """Return one raw action response using DeepSeek JSON Output mode.""" - return self._complete_action_response_once(messages, max_output_tokens=max_output_tokens).content - - def _complete_action_response_once( - self, - messages: list[dict[str, str]], - *, - max_output_tokens: int | None = None, - tools: list[object] | None = None, - hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, - mcp_approval_callback: object | None = None, - ) -> LLMResponse: - """Return one raw action response plus DeepSeek usage metadata.""" - if tools is not None: - try: - return self._complete_native_action_response_once(messages, tools=tools) - except LLMError as exc: - fallback = self._complete_json_action_response_once(with_json_action_prompt(messages)) - fallback.metadata.update( - { - "action_transport": "chulk_json_fallback", - "native_tool_call_error": str(exc), - } - ) - return fallback - return self._complete_json_action_response_once(messages) - - def _complete_json_action_response_once(self, messages: list[dict[str, str]]) -> LLMResponse: - request = { - "model": self.model, - "messages": chat_messages(messages), - "stream": False, - "response_format": {"type": "json_object"}, - } - try: - response = self._client.chat.completions.create(**request) - except Exception as exc: - raise LLMError(f"DeepSeek structured action request failed: {exc}") from exc - - try: - content = response.choices[0].message.content - except (AttributeError, IndexError) as exc: - raise LLMError("DeepSeek structured action response did not include message content") from exc - - if isinstance(content, str) and content: - result = self._response_from_provider(messages, content, getattr(response, "usage", None)) - result.metadata.update({"action_transport": "chulk_json"}) - return result - raise LLMError("DeepSeek structured action response content was empty") - - def _complete_native_action_response_once(self, messages: list[dict[str, str]], *, tools: list[object]) -> LLMResponse: - request = { - "model": self.model, - "messages": chat_messages(messages), - "stream": False, - "tools": chat_completion_tools(tools), - "tool_choice": "auto", - } - try: - response = self._client.chat.completions.create(**request) - except Exception as exc: - raise LLMError(f"DeepSeek native tool action request failed: {exc}") from exc - - message = _response_message(response) - content, raw_tool_call = _normalize_chat_native_action_message(message) - result = self._response_from_provider(messages, content, getattr(response, "usage", None)) - result.metadata.update( - { - "action_transport": "provider_native", - "provider_tool_call": raw_tool_call, - } - ) - return result - - def _response_from_provider(self, messages: list[dict[str, str]], content: str, usage_payload: object) -> LLMResponse: - usage = normalize_deepseek_usage(usage_payload) - if usage is None: - return self._response_with_estimated_usage(messages, content) - return LLMResponse( - content=content, - usage=usage, - cost=estimate_cost("deepseek", self.model, usage), - provider="deepseek", - model=self.model, - ) - - -def _response_message(response: object) -> object: - try: - return response.choices[0].message - except (AttributeError, IndexError) as exc: - raise LLMError("DeepSeek response did not include message content") from exc - - -def _normalize_chat_native_action_message(message: object) -> tuple[str, dict | None]: - tool_calls = _value(message, "tool_calls") - if isinstance(tool_calls, list) and tool_calls: - tool_call = tool_calls[0] - function = _value(tool_call, "function") - name = _value(function, "name") - if not isinstance(name, str) or not name: - raise LLMError("Native tool call did not include a function name") - arguments = parse_native_arguments(_value(function, "arguments")) - payload = native_tool_action_payload(name, arguments) - return action_payload_json(payload), public_value(tool_call) - - content = _value(message, "content") - if isinstance(content, str) and content.strip(): - return action_payload_json(native_final_answer_payload(content.strip())), None - raise LLMError("Native action response did not include a tool call or content") - - -def _value(source: object, key: str) -> object: - if isinstance(source, dict): - return source.get(key) - return getattr(source, key, None) diff --git a/chulk/llm/providers/gemini.py b/chulk/llm/providers/gemini.py new file mode 100644 index 0000000..ec4527b --- /dev/null +++ b/chulk/llm/providers/gemini.py @@ -0,0 +1,840 @@ +"""Native Google Gemini provider client.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from typing import Any, cast + +from chulk.core.actions import STRICT_AGENT_ACTION_JSON_SCHEMA +from chulk.llm.base import ( + LLMClient, + LLMConfigurationError, + LLMError, + LLMErrorCode, + LLMStreamChunk, + is_action_transport_fallback_error, + provider_error_from_exception, +) +from chulk.llm.capabilities import LLMCapabilities +from chulk.llm.pricing import estimate_cost +from chulk.llm.tools import ( + action_payload_json, + native_final_answer_payload, + native_tool_action_payload, + parse_native_arguments, + provider_action_tools, + public_value, + with_json_action_prompt, +) +from chulk.llm.usage import LLMResponse, LLMUsage + + +GEMINI_CAPABILITIES = LLMCapabilities( + supports_structured_output=True, + supports_json_mode=True, + supports_streaming=True, + supports_native_tool_calling=True, + api_style="generate_content", +) + + +class GeminiGenerateContentClient(LLMClient): + """LLM client backed by the native ``google-genai`` GenerateContent API.""" + + capabilities = GEMINI_CAPABILITIES + provider = "gemini" + + def __init__( + self, + *, + model: str, + api_key: str | None = None, + base_url: str | None = None, + timeout_seconds: float = 60.0, + max_retries: int = 2, + client: Any | None = None, + async_client: Any | None = None, + ) -> None: + if not model.strip(): + raise LLMConfigurationError( + "CHULK_MODEL is required for the Gemini LLM client", + provider=self.provider, + ) + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be greater than zero") + if max_retries < 0: + raise ValueError("max_retries cannot be negative") + + self.model = model.strip() + self.base_url = base_url.strip() if base_url is not None else None + self._async_client = async_client + if base_url is not None and not self.base_url: + raise ValueError("base_url must be non-empty when provided") + + if client is not None: + self._client = client + if async_client is None: + self._async_client = getattr(client, "aio", None) + return + + resolved_api_key = api_key.strip() if api_key else "" + if not resolved_api_key: + raise LLMConfigurationError( + "GEMINI_API_KEY or CHULK_GEMINI_API_KEY is required for Gemini", + provider=self.provider, + model=self.model, + ) + + try: + from google import genai + except ImportError as exc: + raise LLMConfigurationError( + "The google-genai package is required. Install it with: " + "pip install -e '.[gemini]'", + provider=self.provider, + model=self.model, + ) from exc + + http_options: dict[str, Any] = { + "timeout": max(1, round(timeout_seconds * 1000)), + "retry_options": {"attempts": max_retries + 1}, + } + if self.base_url is not None: + http_options["base_url"] = self.base_url + self._client = genai.Client( + api_key=resolved_api_key, + http_options=cast(Any, http_options), + ) + self._async_client = async_client or getattr(self._client, "aio", None) + + def complete( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> str: + """Return a text response using Gemini GenerateContent.""" + return self.complete_response(messages, max_output_tokens=max_output_tokens).content + + def complete_response( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + """Return text plus normalized Gemini usage metadata.""" + request = self._request(messages, max_output_tokens=max_output_tokens) + response = self._generate(request, operation="request") + content = _response_text(response) + if not content: + raise self._invalid_response_error("Gemini response did not include text") + return self._response_from_provider(messages, content, _value(response, "usage_metadata")) + + async def acomplete_response( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + """Return text through Gemini's native async GenerateContent API.""" + if self._async_client is None: + return await super().acomplete_response(messages, max_output_tokens=max_output_tokens) + request = self._request(messages, max_output_tokens=max_output_tokens) + response = await self._agenerate(request, operation="request") + content = _response_text(response) + if not content: + raise self._invalid_response_error("Gemini response did not include text") + return self._response_from_provider(messages, content, _value(response, "usage_metadata")) + + def stream_complete( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> Iterator[LLMStreamChunk]: + """Yield normalized text chunks from Gemini's native streaming API.""" + request = self._request(messages, max_output_tokens=max_output_tokens) + try: + stream = self._client.models.generate_content_stream(**request) + except Exception as exc: + error = _gemini_error_from_exception( + exc, + message="Gemini streaming request failed", + model=self.model, + ) + if error is exc: + raise + raise error from exc + + text_parts: list[str] = [] + usage_payload: object = None + finish_reason: str | None = None + try: + for chunk in stream: + usage = _value(chunk, "usage_metadata") + if usage is not None: + usage_payload = usage + current_finish_reason = _finish_reason(chunk) + if current_finish_reason is not None: + finish_reason = current_finish_reason + text = _response_text(chunk, strip=False) + if text: + text_parts.append(text) + yield LLMStreamChunk( + type="text_delta", + text=text, + metadata={"transport": "generate_content"}, + ) + except Exception as exc: + error = _gemini_error_from_exception( + exc, + message="Gemini streaming request failed", + model=self.model, + ) + if error is exc: + raise + raise error from exc + + if not text_parts: + raise self._invalid_response_error("Gemini streaming response did not include text") + + response = self._response_from_provider(messages, "".join(text_parts), usage_payload) + yield LLMStreamChunk( + type="completed", + metadata={ + "transport": "generate_content", + "finish_reason": finish_reason, + }, + usage=response.usage, + cost=response.cost, + ) + + def _complete_action_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> str: + return self._complete_action_response_once( + messages, + max_output_tokens=max_output_tokens, + ).content + + def _complete_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, + ) -> LLMResponse: + if hosted_mcp_servers: + raise LLMError( + "Gemini does not support Chulk hosted MCP tools", + provider=self.provider, + model=self.model, + code="unsupported_feature", + ) + if tools is not None: + try: + return self._complete_native_action_response_once( + messages, + tools=tools, + max_output_tokens=max_output_tokens, + ) + except LLMError as exc: + if not is_action_transport_fallback_error(exc): + raise + fallback = self._complete_json_action_response_once( + with_json_action_prompt(messages), + max_output_tokens=max_output_tokens, + ) + fallback.metadata.update( + { + "action_transport": "chulk_json_fallback", + "native_tool_call_error": str(exc), + } + ) + return fallback + return self._complete_json_action_response_once( + messages, + max_output_tokens=max_output_tokens, + ) + + async def _acomplete_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, + ) -> LLMResponse: + if self._async_client is None: + return await super()._acomplete_action_response_once( + messages, + max_output_tokens=max_output_tokens, + tools=tools, + hosted_mcp_servers=hosted_mcp_servers, + mcp_approval_callback=mcp_approval_callback, + ) + if hosted_mcp_servers: + raise LLMError( + "Gemini does not support Chulk hosted MCP tools", + provider=self.provider, + model=self.model, + code="unsupported_feature", + ) + if tools is not None: + try: + return await self._acomplete_native_action_response_once( + messages, + tools=tools, + max_output_tokens=max_output_tokens, + ) + except LLMError as exc: + if not is_action_transport_fallback_error(exc): + raise + fallback = await self._acomplete_json_action_response_once( + with_json_action_prompt(messages), + max_output_tokens=max_output_tokens, + ) + fallback.metadata.update( + { + "action_transport": "chulk_json_fallback", + "native_tool_call_error": str(exc), + } + ) + return fallback + return await self._acomplete_json_action_response_once( + messages, + max_output_tokens=max_output_tokens, + ) + + def _complete_json_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + request = self._request(messages, max_output_tokens=max_output_tokens) + request["config"].update( + { + "response_mime_type": "application/json", + "response_json_schema": STRICT_AGENT_ACTION_JSON_SCHEMA, + } + ) + response = self._generate(request, operation="structured action request") + content = _response_text(response) + if not content: + raise LLMError( + "Gemini structured action response did not include JSON text", + provider=self.provider, + model=self.model, + code="action_shape_error", + ) + result = self._response_from_provider( + messages, + content, + _value(response, "usage_metadata"), + ) + result.metadata.update({"action_transport": "chulk_json"}) + return result + + async def _acomplete_json_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + request = self._request(messages, max_output_tokens=max_output_tokens) + request["config"].update( + { + "response_mime_type": "application/json", + "response_json_schema": STRICT_AGENT_ACTION_JSON_SCHEMA, + } + ) + response = await self._agenerate(request, operation="structured action request") + content = _response_text(response) + if not content: + raise LLMError( + "Gemini structured action response did not include JSON text", + provider=self.provider, + model=self.model, + code="action_shape_error", + ) + result = self._response_from_provider( + messages, + content, + _value(response, "usage_metadata"), + ) + result.metadata.update({"action_transport": "chulk_json"}) + return result + + def _complete_native_action_response_once( + self, + messages: list[dict[str, str]], + *, + tools: list[object], + max_output_tokens: int | None = None, + ) -> LLMResponse: + request = self._request(messages, max_output_tokens=max_output_tokens) + request["config"].update(_native_tool_config(tools)) + response = self._generate( + request, + operation="native tool action request", + action_transport=True, + ) + content, raw_tool_call = _normalize_native_action_response( + response, + provider=self.provider, + model=self.model, + ) + result = self._response_from_provider( + messages, + content, + _value(response, "usage_metadata"), + ) + result.metadata.update( + { + "action_transport": "provider_native", + "provider_tool_call": raw_tool_call, + } + ) + return result + + async def _acomplete_native_action_response_once( + self, + messages: list[dict[str, str]], + *, + tools: list[object], + max_output_tokens: int | None = None, + ) -> LLMResponse: + request = self._request(messages, max_output_tokens=max_output_tokens) + request["config"].update(_native_tool_config(tools)) + response = await self._agenerate( + request, + operation="native tool action request", + action_transport=True, + ) + content, raw_tool_call = _normalize_native_action_response( + response, + provider=self.provider, + model=self.model, + ) + result = self._response_from_provider( + messages, + content, + _value(response, "usage_metadata"), + ) + result.metadata.update( + { + "action_transport": "provider_native", + "provider_tool_call": raw_tool_call, + } + ) + return result + + def _request( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None, + ) -> dict[str, Any]: + system_instruction, contents = _gemini_messages(messages) + config: dict[str, Any] = {} + if system_instruction: + config["system_instruction"] = system_instruction + output_limit = _validate_max_output_tokens(max_output_tokens) + if output_limit is not None: + config["max_output_tokens"] = output_limit + return { + "model": self.model, + "contents": contents, + "config": config, + } + + def _generate( + self, + request: dict[str, Any], + *, + operation: str, + action_transport: bool = False, + ) -> object: + try: + return self._client.models.generate_content(**request) + except Exception as exc: + error = _gemini_error_from_exception( + exc, + message=f"Gemini {operation} failed", + model=self.model, + action_transport=action_transport, + ) + if error is exc: + raise + raise error from exc + + async def _agenerate( + self, + request: dict[str, Any], + *, + operation: str, + action_transport: bool = False, + ) -> object: + async_client = self._async_client + if async_client is None: + raise RuntimeError("Async Gemini client is not configured") + try: + return await async_client.models.generate_content(**request) + except Exception as exc: + error = _gemini_error_from_exception( + exc, + message=f"Gemini {operation} failed", + model=self.model, + action_transport=action_transport, + ) + if error is exc: + raise + raise error from exc + + def _response_from_provider( + self, + messages: list[dict[str, str]], + content: str, + usage_payload: object, + ) -> LLMResponse: + usage = normalize_gemini_usage(usage_payload) + if usage is None: + estimated = self._response_with_estimated_usage(messages, content) + return LLMResponse( + content=estimated.content, + usage=estimated.usage, + cost=estimated.cost, + provider=self.provider, + model=self.model, + ) + return LLMResponse( + content=content, + usage=usage, + cost=estimate_cost(self.provider, self.model, usage), + provider=self.provider, + model=self.model, + ) + + def _invalid_response_error(self, message: str) -> LLMError: + return LLMError( + message, + provider=self.provider, + model=self.model, + code="invalid_response", + retryable=True, + fallback_eligible=True, + ) + + +def normalize_gemini_usage(usage: object) -> LLMUsage | None: + """Normalize Gemini ``usage_metadata`` into Chulk usage accounting.""" + if usage is None: + return None + input_tokens = _int_value(_value(usage, "prompt_token_count")) + output_tokens = _int_value(_value(usage, "candidates_token_count")) + total_tokens = _int_value(_value(usage, "total_token_count")) or input_tokens + output_tokens + cached_tokens = _int_value(_value(usage, "cached_content_token_count")) + reasoning_tokens = _int_value(_value(usage, "thoughts_token_count")) + if not any([input_tokens, output_tokens, total_tokens, cached_tokens, reasoning_tokens]): + return None + raw = public_value(usage) + return LLMUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + cached_input_tokens=cached_tokens, + cache_hit_input_tokens=cached_tokens, + cache_miss_input_tokens=max(input_tokens - cached_tokens, 0), + reasoning_tokens=reasoning_tokens, + estimated=False, + source="provider", + raw=raw if isinstance(raw, dict) else {}, + ) + + +def _gemini_messages(messages: list[dict[str, str]]) -> tuple[str, list[dict[str, Any]]]: + instruction_parts: list[str] = [] + contents: list[dict[str, Any]] = [] + for message in messages: + role = message.get("role", "") + content = message.get("content", "") + if role in {"system", "developer"}: + if content: + instruction_parts.append(content) + continue + if role == "assistant": + gemini_role = "model" + elif role == "user": + gemini_role = "user" + else: + gemini_role = "user" + content = f"{role or 'message'}: {content}" + _append_content(contents, gemini_role, content) + + if not contents: + contents.append({"role": "user", "parts": [{"text": "Continue."}]}) + return "\n\n".join(instruction_parts), contents + + +def _append_content(contents: list[dict[str, Any]], role: str, text: str) -> None: + if contents and contents[-1]["role"] == role: + current_text = str(contents[-1]["parts"][0]["text"]) + contents[-1]["parts"][0]["text"] = "\n\n".join([current_text, text]) + return + contents.append({"role": role, "parts": [{"text": text}]}) + + +def _native_tool_config(tools: list[object]) -> dict[str, Any]: + declarations = [ + { + "name": declaration["name"], + "description": declaration["description"], + "parameters_json_schema": declaration["parameters"], + } + for declaration in provider_action_tools(tools) + ] + return { + "tools": [{"function_declarations": declarations}], + "tool_config": {"function_calling_config": {"mode": "AUTO"}}, + "automatic_function_calling": {"disable": True}, + } + + +def _normalize_native_action_response( + response: object, + *, + provider: str, + model: str, +) -> tuple[str, dict[str, Any] | None]: + function_calls = _function_calls(response) + if function_calls: + if len(function_calls) != 1: + raise LLMError( + "Gemini native action response included multiple function calls; " + "Chulk accepts one action per turn", + provider=provider, + model=model, + code="action_shape_error", + ) + function_call = function_calls[0] + name = _value(function_call, "name") + if not isinstance(name, str) or not name: + raise LLMError( + "Gemini native function call did not include a function name", + provider=provider, + model=model, + code="action_shape_error", + ) + raw_arguments = _value(function_call, "args") + if not isinstance(raw_arguments, (dict, str)): + public_arguments = public_value(raw_arguments) + raw_arguments = public_arguments if isinstance(public_arguments, dict) else raw_arguments + try: + arguments = parse_native_arguments(raw_arguments) + except ValueError as exc: + raise LLMError( + str(exc), + provider=provider, + model=model, + code="action_shape_error", + ) from exc + payload = native_tool_action_payload(name, arguments) + raw_tool_call = public_value(function_call) + return ( + action_payload_json(payload), + raw_tool_call if isinstance(raw_tool_call, dict) else None, + ) + + text = _response_text(response) + if text: + return action_payload_json(native_final_answer_payload(text)), None + raise LLMError( + "Gemini native action response did not include a function call or text", + provider=provider, + model=model, + code="action_shape_error", + ) + + +def _function_calls(response: object) -> list[object]: + direct = _safe_value(response, "function_calls") + if isinstance(direct, (list, tuple)): + return list(direct) + + calls: list[object] = [] + candidates = _safe_value(response, "candidates") + if not isinstance(candidates, (list, tuple)): + return calls + for candidate in candidates: + content = _safe_value(candidate, "content") + parts = _safe_value(content, "parts") + if not isinstance(parts, (list, tuple)): + continue + for part in parts: + function_call = _safe_value(part, "function_call") + if function_call is not None: + calls.append(function_call) + return calls + + +def _response_text(response: object, *, strip: bool = True) -> str: + direct = _safe_value(response, "text") + if isinstance(direct, str) and direct.strip(): + return direct.strip() if strip else direct + + text_parts: list[str] = [] + candidates = _safe_value(response, "candidates") + if not isinstance(candidates, (list, tuple)): + return "" + for candidate in candidates: + content = _safe_value(candidate, "content") + parts = _safe_value(content, "parts") + if not isinstance(parts, (list, tuple)): + continue + for part in parts: + text = _safe_value(part, "text") + if isinstance(text, str) and text: + text_parts.append(text) + joined = "".join(text_parts) + return joined.strip() if strip else joined + + +def _finish_reason(response: object) -> str | None: + candidates = _safe_value(response, "candidates") + if not isinstance(candidates, (list, tuple)) or not candidates: + return None + finish_reason = _safe_value(candidates[0], "finish_reason") + if finish_reason is None: + return None + value = getattr(finish_reason, "value", finish_reason) + return str(value) + + +def _gemini_error_from_exception( + exc: Exception, + *, + message: str, + model: str, + action_transport: bool = False, +) -> LLMError: + status_code = _gemini_status_code(exc) + error_text = str(exc).lower() + if isinstance(exc, LLMError): + return provider_error_from_exception( + exc, + message=message, + provider="gemini", + model=model, + action_transport=action_transport, + ) + + if _has_any(error_text, "api key not valid", "invalid api key", "api_key_invalid"): + classification: tuple[LLMErrorCode, bool, bool] = ( + "authentication_error", + False, + False, + ) + elif action_transport and status_code in {None, 400, 409, 422} and _has_any( + error_text, + "function calling is not supported", + "function calls are not supported", + "tools are not supported", + "unsupported function calling", + ): + classification = ("unsupported_feature", False, True) + else: + normalized = provider_error_from_exception( + exc, + message=message, + provider="gemini", + model=model, + action_transport=action_transport, + ) + if normalized.code != "unknown": + return normalized + + if status_code == 401: + classification = ("authentication_error", False, False) + elif status_code == 403: + classification = ("permission_denied", False, False) + elif status_code == 404 or _has_any( + error_text, + "model not found", + "unknown model", + "model does not exist", + ): + classification = ("model_not_found", False, False) + elif status_code == 429: + classification = ("rate_limit", True, True) + elif status_code == 408: + classification = ("timeout", True, True) + elif status_code is not None and status_code >= 500: + classification = ("server_error", True, True) + elif status_code in {400, 409, 422}: + classification = ("invalid_request", False, False) + else: + return normalized + + code, retryable, fallback_eligible = classification + return LLMError( + f"{message}: {exc}", + provider="gemini", + model=model, + code=code, + retryable=retryable, + fallback_eligible=fallback_eligible, + ) + + +def _gemini_status_code(exc: Exception) -> int | None: + for value in ( + getattr(exc, "status_code", None), + getattr(exc, "code", None), + getattr(getattr(exc, "response", None), "status_code", None), + ): + if isinstance(value, int): + return value + return None + + +def _has_any(value: str, *markers: str) -> bool: + return any(marker in value for marker in markers) + + +def _validate_max_output_tokens(value: int | None) -> int | None: + if value is None: + return None + if isinstance(value, bool) or value < 1: + raise ValueError("max_output_tokens must be greater than zero") + return value + + +def _safe_value(source: object, key: str) -> object: + try: + return _value(source, key) + except Exception: + return None + + +def _value(source: object, key: str) -> object: + if isinstance(source, dict): + return source.get(key) + return getattr(source, key, None) + + +def _int_value(value: object) -> int: + if value is None: + return 0 + if not isinstance(value, (str, bytes, bytearray, int, float)): + return 0 + try: + return max(0, int(value)) + except (TypeError, ValueError): + return 0 diff --git a/chulk/llm/providers/local.py b/chulk/llm/providers/local.py index 70f15e9..951d07e 100644 --- a/chulk/llm/providers/local.py +++ b/chulk/llm/providers/local.py @@ -4,19 +4,13 @@ from typing import Any -from chulk.llm.base import LLMClient, LLMConfigurationError, LLMError from chulk.llm.capabilities import LLMCapabilities from chulk.llm.messages import local_chat_messages -from chulk.llm.tools import ( - action_payload_json, - chat_completion_tools, - native_final_answer_payload, - native_tool_action_payload, - parse_native_arguments, - public_value, - with_json_action_prompt, +from chulk.llm.providers.chat_completions import ( + ChatCompletionsTransportProfile, + OpenAICompatibleChatCompletionsClient, ) -from chulk.llm.usage import LLMResponse +from chulk.llm.usage import normalize_chat_completions_usage DEFAULT_LOCAL_BASE_URL = "http://localhost:1234/v1" @@ -25,14 +19,23 @@ LOCAL_CAPABILITIES = LLMCapabilities( supports_structured_output=False, supports_json_mode=False, - supports_streaming=False, + supports_streaming=True, supports_native_tool_calling=True, api_style="chat_completions", ) +LOCAL_TRANSPORT_PROFILE = ChatCompletionsTransportProfile( + provider="local", + display_name="Local LLM", + capabilities=LOCAL_CAPABILITIES, + normalize_messages=local_chat_messages, + normalize_usage=normalize_chat_completions_usage, + default_api_key=DEFAULT_LOCAL_API_KEY, +) + -class LocalOpenAICompatibleClient(LLMClient): - """LLM client backed by a local OpenAI-compatible chat-completions server.""" +class LocalOpenAICompatibleClient(OpenAICompatibleChatCompletionsClient): + """LLM client backed by a local OpenAI-compatible Chat Completions server.""" capabilities = LOCAL_CAPABILITIES provider = "local" @@ -46,149 +49,15 @@ def __init__( timeout_seconds: float = 60.0, max_retries: int = 2, client: Any | None = None, + async_client: Any | None = None, ) -> None: - self.model = model - self.base_url = _validate_base_url(base_url) - - if client is not None: - self._client = client - return - - try: - from openai import OpenAI - except ImportError as exc: - raise LLMConfigurationError( - "The openai package is required. Install it with: pip install -e '.[openai]'" - ) from exc - - self._client = OpenAI( - api_key=api_key or DEFAULT_LOCAL_API_KEY, - base_url=self.base_url, - timeout=timeout_seconds, + super().__init__( + profile=LOCAL_TRANSPORT_PROFILE, + model=model, + api_key=api_key, + base_url=base_url, + timeout_seconds=timeout_seconds, max_retries=max_retries, + client=client, + async_client=async_client, ) - - def complete(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> str: - """Return a text response using a local OpenAI-compatible chat endpoint.""" - return self.complete_response(messages, max_output_tokens=max_output_tokens).content - - def complete_response( - self, - messages: list[dict[str, str]], - *, - max_output_tokens: int | None = None, - ) -> LLMResponse: - """Return a text response with estimated token usage.""" - request = { - "model": self.model, - "messages": local_chat_messages(messages), - "stream": False, - } - try: - response = self._client.chat.completions.create(**request) - except Exception as exc: - raise LLMError(f"Local LLM request failed: {exc}") from exc - - try: - content = response.choices[0].message.content - except (AttributeError, IndexError) as exc: - raise LLMError("Local LLM response did not include message content") from exc - - if isinstance(content, str) and content: - return self._response_with_estimated_usage(messages, content) - raise LLMError("Local LLM response content was empty") - - def _complete_action_once(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> str: - """Return one raw action response from the local model.""" - return self.complete(messages) - - def _complete_action_response_once( - self, - messages: list[dict[str, str]], - *, - max_output_tokens: int | None = None, - tools: list[object] | None = None, - hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, - mcp_approval_callback: object | None = None, - ) -> LLMResponse: - """Return one raw action response, using native local tool calls by default.""" - if tools is not None: - try: - return self._complete_native_action_response_once(messages, tools=tools) - except LLMError as exc: - fallback = self._complete_json_action_response_once(with_json_action_prompt(messages)) - fallback.metadata.update( - { - "action_transport": "chulk_json_fallback", - "native_tool_call_error": str(exc), - } - ) - return fallback - return self._complete_json_action_response_once(messages) - - def _complete_json_action_response_once(self, messages: list[dict[str, str]]) -> LLMResponse: - content = self.complete(messages) - response = self._response_with_estimated_usage(messages, content) - response.metadata.update({"action_transport": "chulk_json"}) - return response - - def _complete_native_action_response_once(self, messages: list[dict[str, str]], *, tools: list[object]) -> LLMResponse: - request = { - "model": self.model, - "messages": local_chat_messages(messages), - "stream": False, - "tools": chat_completion_tools(tools), - "tool_choice": "auto", - } - try: - response = self._client.chat.completions.create(**request) - except Exception as exc: - raise LLMError(f"Local native tool action request failed: {exc}") from exc - - message = _response_message(response) - content, raw_tool_call = _normalize_chat_native_action_message(message) - result = self._response_with_estimated_usage(messages, content) - result.metadata.update( - { - "action_transport": "provider_native", - "provider_tool_call": raw_tool_call, - } - ) - return result - - -def _validate_base_url(value: str) -> str: - if not value.strip(): - raise ValueError("base_url must be non-empty") - return value.strip() - - -def _response_message(response: object) -> object: - try: - return response.choices[0].message - except (AttributeError, IndexError) as exc: - raise LLMError("Local LLM response did not include message content") from exc - - -def _normalize_chat_native_action_message(message: object) -> tuple[str, dict | None]: - tool_calls = _value(message, "tool_calls") - if isinstance(tool_calls, list) and tool_calls: - tool_call = tool_calls[0] - function = _value(tool_call, "function") - name = _value(function, "name") - if not isinstance(name, str) or not name: - raise LLMError("Native tool call did not include a function name") - arguments = parse_native_arguments(_value(function, "arguments")) - payload = native_tool_action_payload(name, arguments) - return action_payload_json(payload), public_value(tool_call) - - content = _value(message, "content") - if isinstance(content, str) and content.strip(): - return action_payload_json(native_final_answer_payload(content.strip())), None - raise LLMError("Native action response did not include a tool call or content") - - -def _value(source: object, key: str) -> object: - if isinstance(source, dict): - return source.get(key) - return getattr(source, key, None) diff --git a/chulk/llm/providers/openai.py b/chulk/llm/providers/openai.py index e66c82e..d7a6b79 100644 --- a/chulk/llm/providers/openai.py +++ b/chulk/llm/providers/openai.py @@ -6,7 +6,14 @@ from typing import Any from chulk.core.actions import STRICT_AGENT_ACTION_JSON_SCHEMA -from chulk.llm.base import LLMClient, LLMConfigurationError, LLMError, LLMStreamChunk +from chulk.llm.base import ( + LLMClient, + LLMConfigurationError, + LLMError, + LLMStreamChunk, + is_action_transport_fallback_error, + provider_error_from_exception, +) from chulk.llm.capabilities import LLMCapabilities from chulk.llm.messages import split_instructions from chulk.llm.pricing import estimate_cost @@ -47,22 +54,30 @@ def __init__( max_retries: int = 2, max_output_tokens: int | None = None, client: Any | None = None, + async_client: Any | None = None, ) -> None: self.model = model self.max_output_tokens = _validate_max_output_tokens(max_output_tokens) + self._async_client = async_client if client is not None: self._client = client return if not api_key: - raise LLMConfigurationError("OPENAI_API_KEY is required for the OpenAI LLM client") + raise LLMConfigurationError( + "OPENAI_API_KEY is required for the OpenAI LLM client", + provider=self.provider, + model=self.model, + ) try: from openai import OpenAI except ImportError as exc: raise LLMConfigurationError( - "The openai package is required. Install it with: pip install -e '.[openai]'" + "The openai package is required. Install it with: pip install -e '.[openai]'", + provider=self.provider, + model=self.model, ) from exc self._client = OpenAI( @@ -70,6 +85,19 @@ def __init__( timeout=timeout_seconds, max_retries=max_retries, ) + if async_client is not None: + self._async_client = async_client + else: + try: + from openai import AsyncOpenAI + except ImportError: + self._async_client = None + else: + self._async_client = AsyncOpenAI( + api_key=api_key, + timeout=timeout_seconds, + max_retries=max_retries, + ) def complete(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> str: """Return a text response using OpenAI's Responses API.""" @@ -86,12 +114,51 @@ def complete_response( try: response = self._client.responses.create(**request) except Exception as exc: - raise LLMError(f"OpenAI request failed: {exc}") from exc + error = provider_error_from_exception( + exc, + message="OpenAI request failed", + provider=self.provider, + model=self.model, + ) + if error is exc: + raise + raise error from exc output_text = getattr(response, "output_text", None) if isinstance(output_text, str) and output_text: return self._response_from_provider(messages, output_text, getattr(response, "usage", None)) - raise LLMError("OpenAI response did not include output_text") + raise self._invalid_response_error("OpenAI response did not include output_text") + + async def acomplete_response( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + """Return text through the native async Responses API.""" + if self._async_client is None: + return await super().acomplete_response(messages, max_output_tokens=max_output_tokens) + request = self._text_request(messages, max_output_tokens=max_output_tokens) + async_client = self._async_client + if async_client is None: + raise RuntimeError("Async OpenAI client is not configured") + try: + response = await async_client.responses.create(**request) + except Exception as exc: + error = provider_error_from_exception( + exc, + message="OpenAI request failed", + provider=self.provider, + model=self.model, + ) + if error is exc: + raise + raise error from exc + + output_text = getattr(response, "output_text", None) + if isinstance(output_text, str) and output_text: + return self._response_from_provider(messages, output_text, getattr(response, "usage", None)) + raise self._invalid_response_error("OpenAI response did not include output_text") def stream_complete( self, @@ -105,29 +172,55 @@ def stream_complete( try: stream = self._client.responses.create(**request) except Exception as exc: - raise LLMError(f"OpenAI streaming request failed: {exc}") from exc + error = provider_error_from_exception( + exc, + message="OpenAI streaming request failed", + provider=self.provider, + model=self.model, + ) + if error is exc: + raise + raise error from exc saw_text = False completed = False usage = None - for event in stream: - event_type = _event_value(event, "type") - if event_type == "response.output_text.delta": - delta = _event_value(event, "delta") - if isinstance(delta, str) and delta: - saw_text = True - yield LLMStreamChunk(type="text_delta", text=delta, metadata={"event_type": event_type}) - continue - if event_type == "response.completed": - completed = True - completed_response = _event_value(event, "response") - usage = normalize_openai_usage(_event_value(completed_response, "usage")) - continue - if event_type == "error": - raise LLMError(f"OpenAI streaming request failed: {_event_error_message(event)}") + try: + for event in stream: + event_type = _event_value(event, "type") + if event_type == "response.output_text.delta": + delta = _event_value(event, "delta") + if isinstance(delta, str) and delta: + saw_text = True + yield LLMStreamChunk(type="text_delta", text=delta, metadata={"event_type": event_type}) + continue + if event_type == "response.completed": + completed = True + completed_response = _event_value(event, "response") + usage = normalize_openai_usage(_event_value(completed_response, "usage")) + continue + if event_type == "error": + raise LLMError( + f"OpenAI streaming request failed: {_event_error_message(event)}", + provider=self.provider, + model=self.model, + code="server_error", + retryable=True, + fallback_eligible=True, + ) + except Exception as exc: + error = provider_error_from_exception( + exc, + message="OpenAI streaming request failed", + provider=self.provider, + model=self.model, + ) + if error is exc: + raise + raise error from exc if not saw_text: - raise LLMError("OpenAI streaming response did not include output text") + raise self._invalid_response_error("OpenAI streaming response did not include output text") if completed: cost = estimate_cost("openai", self.model, usage) if usage is not None else None yield LLMStreamChunk( @@ -165,6 +258,8 @@ def _complete_action_response_once( except LLMError as exc: if hosted_mcp_servers: raise + if not is_action_transport_fallback_error(exc): + raise fallback = self._complete_json_action_response_once( with_json_action_prompt(messages), max_output_tokens=max_output_tokens, @@ -178,6 +273,53 @@ def _complete_action_response_once( return fallback return self._complete_json_action_response_once(messages, max_output_tokens=max_output_tokens) + async def _acomplete_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, + ) -> LLMResponse: + if self._async_client is None: + return await super()._acomplete_action_response_once( + messages, + max_output_tokens=max_output_tokens, + tools=tools, + hosted_mcp_servers=hosted_mcp_servers, + mcp_approval_callback=mcp_approval_callback, + ) + if tools is not None: + try: + return await self._acomplete_native_action_response_once( + messages, + tools=tools, + max_output_tokens=max_output_tokens, + hosted_mcp_servers=hosted_mcp_servers, + mcp_approval_callback=mcp_approval_callback, + ) + except LLMError as exc: + if hosted_mcp_servers: + raise + if not is_action_transport_fallback_error(exc): + raise + fallback = await self._acomplete_json_action_response_once( + with_json_action_prompt(messages), + max_output_tokens=max_output_tokens, + ) + fallback.metadata.update( + { + "action_transport": "chulk_json_fallback", + "native_tool_call_error": str(exc), + } + ) + return fallback + return await self._acomplete_json_action_response_once( + messages, + max_output_tokens=max_output_tokens, + ) + def _complete_json_action_response_once( self, messages: list[dict[str, str]], @@ -185,7 +327,7 @@ def _complete_json_action_response_once( max_output_tokens: int | None = None, ) -> LLMResponse: instructions, response_input = split_instructions(messages) - request = { + request: dict[str, Any] = { "model": self.model, "instructions": instructions or None, "input": response_input, @@ -204,14 +346,68 @@ def _complete_json_action_response_once( try: response = self._client.responses.create(**request) except Exception as exc: - raise LLMError(f"OpenAI structured action request failed: {exc}") from exc + error = provider_error_from_exception( + exc, + message="OpenAI structured action request failed", + provider=self.provider, + model=self.model, + ) + if error is exc: + raise + raise error from exc output_text = getattr(response, "output_text", None) if isinstance(output_text, str) and output_text: result = self._response_from_provider(messages, output_text, getattr(response, "usage", None)) result.metadata.update({"action_transport": "chulk_json"}) return result - raise LLMError("OpenAI structured action response did not include output_text") + raise self._invalid_response_error("OpenAI structured action response did not include output_text") + + async def _acomplete_json_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + instructions, response_input = split_instructions(messages) + request: dict[str, Any] = { + "model": self.model, + "instructions": instructions or None, + "input": response_input, + "text": { + "format": { + "type": "json_schema", + "name": "agent_action", + "strict": True, + "schema": STRICT_AGENT_ACTION_JSON_SCHEMA, + } + }, + } + output_limit = _request_max_output_tokens(self.max_output_tokens, max_output_tokens) + if output_limit is not None: + request["max_output_tokens"] = output_limit + async_client = self._async_client + if async_client is None: + raise RuntimeError("Async OpenAI client is not configured") + try: + response = await async_client.responses.create(**request) + except Exception as exc: + error = provider_error_from_exception( + exc, + message="OpenAI structured action request failed", + provider=self.provider, + model=self.model, + ) + if error is exc: + raise + raise error from exc + + output_text = getattr(response, "output_text", None) + if isinstance(output_text, str) and output_text: + result = self._response_from_provider(messages, output_text, getattr(response, "usage", None)) + result.metadata.update({"action_transport": "chulk_json"}) + return result + raise self._invalid_response_error("OpenAI structured action response did not include output_text") def _complete_native_action_response_once( self, @@ -223,7 +419,7 @@ def _complete_native_action_response_once( mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, ) -> LLMResponse: instructions, response_input = split_instructions(messages) - request = { + request: dict[str, Any] = { "model": self.model, "instructions": instructions or None, "input": response_input, @@ -238,7 +434,52 @@ def _complete_native_action_response_once( mcp_approval_callback=mcp_approval_callback, ) - content, metadata = _normalize_openai_native_action_response(response) + content, metadata = _normalize_openai_native_action_response( + response, + provider=self.provider, + model=self.model, + ) + result = self._response_from_provider(messages, content, getattr(response, "usage", None)) + result.metadata.update( + { + "action_transport": "provider_native", + "provider_tool_call": metadata.get("provider_tool_call"), + "provider_mcp_output": metadata.get("provider_mcp_output", []), + "provider_mcp_approval": approval_metadata, + } + ) + return result + + async def _acomplete_native_action_response_once( + self, + messages: list[dict[str, str]], + *, + tools: list[object], + max_output_tokens: int | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None = None, + ) -> LLMResponse: + instructions, response_input = split_instructions(messages) + request: dict[str, Any] = { + "model": self.model, + "instructions": instructions or None, + "input": response_input, + "tools": openai_response_tools(tools, hosted_mcp_servers=hosted_mcp_servers), + "tool_choice": "auto", + } + output_limit = _request_max_output_tokens(self.max_output_tokens, max_output_tokens) + if output_limit is not None: + request["max_output_tokens"] = output_limit + response, approval_metadata = await self._acreate_native_action_response( + request, + mcp_approval_callback=mcp_approval_callback, + ) + + content, metadata = _normalize_openai_native_action_response( + response, + provider=self.provider, + model=self.model, + ) result = self._response_from_provider(messages, content, getattr(response, "usage", None)) result.metadata.update( { @@ -262,18 +503,37 @@ def _create_native_action_response( try: response = self._client.responses.create(**current_request) except Exception as exc: - raise LLMError(f"OpenAI native tool action request failed: {exc}") from exc + error = provider_error_from_exception( + exc, + message="OpenAI native tool action request failed", + provider=self.provider, + model=self.model, + action_transport=True, + ) + if error is exc: + raise + raise error from exc approval_request = _find_mcp_approval_request(response) if approval_request is None: return response, approval_metadata if mcp_approval_callback is None: - raise LLMError("OpenAI MCP approval request could not be handled without a permission callback") + raise LLMError( + "OpenAI MCP approval request could not be handled without a permission callback", + provider=self.provider, + model=self.model, + code="configuration_error", + ) approval_payload = public_value(approval_request) approval_id = _mcp_approval_request_id(approval_payload) if not approval_id: - raise LLMError("OpenAI MCP approval request did not include an approval id") + raise LLMError( + "OpenAI MCP approval request did not include an approval id", + provider=self.provider, + model=self.model, + code="invalid_response", + ) approved = bool(mcp_approval_callback(approval_payload)) approval_metadata.append( { @@ -285,7 +545,11 @@ def _create_native_action_response( ) current_request = { "model": self.model, - "previous_response_id": _response_id(response), + "previous_response_id": _response_id( + response, + provider=self.provider, + model=self.model, + ), "input": [ { "type": "mcp_approval_response", @@ -299,7 +563,94 @@ def _create_native_action_response( if request.get("max_output_tokens") is not None: current_request["max_output_tokens"] = request["max_output_tokens"] - raise LLMError("OpenAI MCP approval loop exceeded the maximum continuation count") + raise LLMError( + "OpenAI MCP approval loop exceeded the maximum continuation count", + provider=self.provider, + model=self.model, + code="invalid_response", + ) + + async def _acreate_native_action_response( + self, + request: dict[str, Any], + *, + mcp_approval_callback: Callable[[dict[str, Any]], bool] | None, + ) -> tuple[object, list[dict[str, Any]]]: + async_client = self._async_client + if async_client is None: + raise RuntimeError("Async OpenAI client is not configured") + approval_metadata: list[dict[str, Any]] = [] + current_request = dict(request) + for _ in range(4): + try: + response = await async_client.responses.create(**current_request) + except Exception as exc: + error = provider_error_from_exception( + exc, + message="OpenAI native tool action request failed", + provider=self.provider, + model=self.model, + action_transport=True, + ) + if error is exc: + raise + raise error from exc + + approval_request = _find_mcp_approval_request(response) + if approval_request is None: + return response, approval_metadata + if mcp_approval_callback is None: + raise LLMError( + "OpenAI MCP approval request could not be handled without a permission callback", + provider=self.provider, + model=self.model, + code="configuration_error", + ) + + approval_payload = public_value(approval_request) + approval_id = _mcp_approval_request_id(approval_payload) + if not approval_id: + raise LLMError( + "OpenAI MCP approval request did not include an approval id", + provider=self.provider, + model=self.model, + code="invalid_response", + ) + approved = bool(mcp_approval_callback(approval_payload)) + approval_metadata.append( + { + "approval_request_id": approval_id, + "server_label": approval_payload.get("server_label"), + "name": approval_payload.get("name"), + "approved": approved, + } + ) + current_request = { + "model": self.model, + "previous_response_id": _response_id( + response, + provider=self.provider, + model=self.model, + ), + "input": [ + { + "type": "mcp_approval_response", + "approval_request_id": approval_id, + "approve": approved, + } + ], + "tools": request["tools"], + "tool_choice": "auto", + } + if request.get("max_output_tokens") is not None: + current_request["max_output_tokens"] = request["max_output_tokens"] + + raise LLMError( + "OpenAI MCP approval loop exceeded the maximum continuation count", + provider=self.provider, + model=self.model, + code="invalid_response", + ) def _text_request(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> dict[str, Any]: instructions, response_input = split_instructions(messages) @@ -325,6 +676,16 @@ def _response_from_provider(self, messages: list[dict[str, str]], content: str, model=self.model, ) + def _invalid_response_error(self, message: str) -> LLMError: + return LLMError( + message, + provider=self.provider, + model=self.model, + code="invalid_response", + retryable=True, + fallback_eligible=True, + ) + def _validate_max_output_tokens(value: int | None) -> int | None: if value is None: @@ -360,7 +721,12 @@ def _event_error_message(event: object) -> str: return str(error or event) -def _normalize_openai_native_action_response(response: object) -> tuple[str, dict[str, Any]]: +def _normalize_openai_native_action_response( + response: object, + *, + provider: str, + model: str, +) -> tuple[str, dict[str, Any]]: metadata: dict[str, Any] = {"provider_tool_call": None, "provider_mcp_output": []} output = _event_value(response, "output") if isinstance(output, list): @@ -373,8 +739,21 @@ def _normalize_openai_native_action_response(response: object) -> tuple[str, dic continue name = _event_value(item, "name") if not isinstance(name, str) or not name: - raise LLMError("OpenAI native tool call did not include a function name") - arguments = parse_native_arguments(_event_value(item, "arguments")) + raise LLMError( + "OpenAI native tool call did not include a function name", + provider=provider, + model=model, + code="action_shape_error", + ) + try: + arguments = parse_native_arguments(_event_value(item, "arguments")) + except ValueError as exc: + raise LLMError( + str(exc), + provider=provider, + model=model, + code="action_shape_error", + ) from exc payload = native_tool_action_payload(name, arguments) metadata["provider_tool_call"] = public_value(item) return action_payload_json(payload), metadata @@ -382,7 +761,12 @@ def _normalize_openai_native_action_response(response: object) -> tuple[str, dic output_text = _event_value(response, "output_text") if isinstance(output_text, str) and output_text.strip(): return action_payload_json(native_final_answer_payload(output_text.strip())), metadata - raise LLMError("OpenAI native action response did not include a function call or output_text") + raise LLMError( + "OpenAI native action response did not include a function call or output_text", + provider=provider, + model=model, + code="action_shape_error", + ) def _find_mcp_approval_request(response: object) -> object | None: @@ -403,8 +787,13 @@ def _mcp_approval_request_id(payload: dict[str, Any]) -> str | None: return None -def _response_id(response: object) -> str: +def _response_id(response: object, *, provider: str, model: str) -> str: response_id = _event_value(response, "id") if isinstance(response_id, str) and response_id: return response_id - raise LLMError("OpenAI response did not include an id for MCP approval continuation") + raise LLMError( + "OpenAI response did not include an id for MCP approval continuation", + provider=provider, + model=model, + code="invalid_response", + ) diff --git a/chulk/llm/public.py b/chulk/llm/public.py index d2a9e79..c177355 100644 --- a/chulk/llm/public.py +++ b/chulk/llm/public.py @@ -2,13 +2,21 @@ from __future__ import annotations -from collections.abc import Callable, Iterator +from collections.abc import Awaitable, Callable, Iterator from dataclasses import dataclass, field import time from typing import TYPE_CHECKING, Literal, Protocol -from chulk.llm.base import LLMActionResult, LLMClient, LLMError, LLMStreamChunk -from chulk.llm.factory import create_llm_client +from chulk.llm.base import ( + LLMActionResult, + LLMClient, + LLMError, + LLMStreamChunk, + call_async_with_supported_kwargs, + call_with_supported_kwargs, +) +from chulk.llm.capabilities import LLMModelCapabilities, conservative_model_capabilities +from chulk.llm.factory import create_llm_client, provider_connection_from_config from chulk.llm.usage import LLMCost, LLMResponse, LLMUsage if TYPE_CHECKING: @@ -21,8 +29,13 @@ class BindableLLM(Protocol): """Provider spec that can create an LLM client from runtime config.""" - provider: str - model: str + @property + def provider(self) -> str: + """Provider registry name.""" + + @property + def model(self) -> str: + """Configured model identifier.""" def bind_config(self, config: "Config") -> LLMClient: """Return a configured LLM client.""" @@ -39,12 +52,11 @@ class OpenAIProvider: provider: str = "openai" def bind_config(self, config: "Config") -> LLMClient: + connection = provider_connection_from_config(self.provider, config).with_overrides(api_key=self.api_key or None) return create_llm_client( provider=self.provider, model=self.model, - openai_api_key=self.api_key or config.openai_api_key, - deepseek_api_key=config.deepseek_api_key, - deepseek_base_url=config.deepseek_base_url, + connection=connection, timeout_seconds=self.timeout_seconds or config.llm_timeout_seconds, max_retries=self.max_retries if self.max_retries is not None else config.llm_max_retries, ) @@ -62,12 +74,14 @@ class DeepSeekProvider: provider: str = "deepseek" def bind_config(self, config: Config) -> LLMClient: + connection = provider_connection_from_config(self.provider, config).with_overrides( + api_key=self.api_key or None, + base_url=self.base_url or None, + ) return create_llm_client( provider=self.provider, model=self.model, - openai_api_key=config.openai_api_key, - deepseek_api_key=self.api_key or config.deepseek_api_key, - deepseek_base_url=self.base_url or config.deepseek_base_url, + connection=connection, timeout_seconds=self.timeout_seconds or config.llm_timeout_seconds, max_retries=self.max_retries if self.max_retries is not None else config.llm_max_retries, ) @@ -85,14 +99,139 @@ class LocalProvider: provider: str = "local" def bind_config(self, config: Config) -> LLMClient: + connection = provider_connection_from_config(self.provider, config).with_overrides( + api_key=self.api_key or None, + base_url=self.base_url or None, + ) + return create_llm_client( + provider=self.provider, + model=self.model, + connection=connection, + timeout_seconds=self.timeout_seconds or config.llm_timeout_seconds, + max_retries=self.max_retries if self.max_retries is not None else config.llm_max_retries, + ) + + +@dataclass(frozen=True) +class OpenAICompatibleProvider: + """User-selected hosted OpenAI-compatible provider spec.""" + + model: str + api_key: str | None = None + base_url: str | None = None + timeout_seconds: float | None = None + max_retries: int | None = None + provider: str = "openai-compatible" + + def bind_config(self, config: Config) -> LLMClient: + connection = provider_connection_from_config(self.provider, config).with_overrides( + api_key=self.api_key or None, + base_url=self.base_url or None, + ) + return create_llm_client( + provider=self.provider, + model=self.model, + connection=connection, + timeout_seconds=self.timeout_seconds or config.llm_timeout_seconds, + max_retries=self.max_retries if self.max_retries is not None else config.llm_max_retries, + ) + + +@dataclass(frozen=True) +class OpenRouterProvider: + """OpenRouter provider spec for the public API.""" + + model: str + api_key: str | None = None + base_url: str | None = None + timeout_seconds: float | None = None + max_retries: int | None = None + provider: str = "openrouter" + + def bind_config(self, config: Config) -> LLMClient: + connection = provider_connection_from_config(self.provider, config).with_overrides( + api_key=self.api_key or None, + base_url=self.base_url or None, + ) + return create_llm_client( + provider=self.provider, + model=self.model, + connection=connection, + timeout_seconds=self.timeout_seconds or config.llm_timeout_seconds, + max_retries=self.max_retries if self.max_retries is not None else config.llm_max_retries, + ) + + +@dataclass(frozen=True) +class AnthropicProvider: + """Anthropic provider spec for the public API.""" + + model: str + api_key: str | None = None + base_url: str | None = None + timeout_seconds: float | None = None + max_retries: int | None = None + provider: str = "anthropic" + + def bind_config(self, config: Config) -> LLMClient: + connection = provider_connection_from_config(self.provider, config).with_overrides( + api_key=self.api_key or None, + base_url=self.base_url or None, + ) + return create_llm_client( + provider=self.provider, + model=self.model, + connection=connection, + timeout_seconds=self.timeout_seconds or config.llm_timeout_seconds, + max_retries=self.max_retries if self.max_retries is not None else config.llm_max_retries, + ) + + +@dataclass(frozen=True) +class BedrockProvider: + """AWS Bedrock OpenAI-compatible provider spec for the public API.""" + + model: str + api_key: str | None = None + base_url: str | None = None + timeout_seconds: float | None = None + max_retries: int | None = None + provider: str = "bedrock" + + def bind_config(self, config: Config) -> LLMClient: + connection = provider_connection_from_config(self.provider, config).with_overrides( + api_key=self.api_key or None, + base_url=self.base_url or None, + ) + return create_llm_client( + provider=self.provider, + model=self.model, + connection=connection, + timeout_seconds=self.timeout_seconds or config.llm_timeout_seconds, + max_retries=self.max_retries if self.max_retries is not None else config.llm_max_retries, + ) + + +@dataclass(frozen=True) +class GeminiProvider: + """Google Gemini provider spec for the public API.""" + + model: str + api_key: str | None = None + base_url: str | None = None + timeout_seconds: float | None = None + max_retries: int | None = None + provider: str = "gemini" + + def bind_config(self, config: Config) -> LLMClient: + connection = provider_connection_from_config(self.provider, config).with_overrides( + api_key=self.api_key or None, + base_url=self.base_url or None, + ) return create_llm_client( provider=self.provider, model=self.model, - openai_api_key=config.openai_api_key, - deepseek_api_key=config.deepseek_api_key, - deepseek_base_url=config.deepseek_base_url, - local_api_key=self.api_key or config.local_api_key, - local_base_url=self.base_url or config.local_base_url, + connection=connection, timeout_seconds=self.timeout_seconds or config.llm_timeout_seconds, max_retries=self.max_retries if self.max_retries is not None else config.llm_max_retries, ) @@ -109,6 +248,9 @@ class ProviderAttempt: error: str | None = None usage: LLMUsage | None = None cost: LLMCost | None = None + error_code: str | None = None + retryable: bool | None = None + fallback_eligible: bool | None = None def to_dict(self) -> dict: return { @@ -117,6 +259,9 @@ def to_dict(self) -> dict: "success": self.success, "latency_seconds": self.latency_seconds, "error": self.error, + "error_code": self.error_code, + "retryable": self.retryable, + "fallback_eligible": self.fallback_eligible, "usage": self.usage.to_dict() if self.usage is not None else None, "cost": self.cost.to_dict() if self.cost is not None else None, } @@ -132,12 +277,17 @@ class FallbackChain(LLMClient): last_attempts: list[ProviderAttempt] = field(default_factory=list) last_success_provider: LLMClient | None = field(default=None, init=False, repr=False) _action_attempts: list[ProviderAttempt] | None = field(default=None, init=False, repr=False) + model_capabilities: LLMModelCapabilities | None = field(default=None, init=False) def __post_init__(self) -> None: if not self.providers: raise ValueError("FallbackChain requires at least one provider") if self.strategy != "first_success": raise NotImplementedError(f"FallbackChain strategy is not implemented yet: {self.strategy}") + records = [getattr(provider, "model_capabilities", None) for provider in self.providers] + typed_records = [record for record in records if isinstance(record, LLMModelCapabilities)] + if records and len(typed_records) == len(records): + self.model_capabilities = conservative_model_capabilities(typed_records) def bind_config(self, config: "Config") -> "FallbackChain": bound: list[LLMClient] = [] @@ -148,7 +298,7 @@ def bind_config(self, config: "Config") -> "FallbackChain": bound.append(provider.bind_config(config)) else: raise TypeError(f"Unsupported fallback provider: {provider!r}") - return FallbackChain(bound, strategy=self.strategy) + return FallbackChain(providers=list(bound), strategy=self.strategy) def complete(self, messages: list[dict[str, str]], *, max_output_tokens: int | None = None) -> str: return self.complete_response(messages, max_output_tokens=max_output_tokens).content @@ -163,6 +313,20 @@ def complete_response( lambda provider: _complete_response(provider, messages, max_output_tokens=max_output_tokens) ) + async def acomplete_response( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + ) -> LLMResponse: + return await self._atry_provider_responses( + lambda provider: _acomplete_response( + provider, + messages, + max_output_tokens=max_output_tokens, + ) + ) + def complete_action( self, messages: list[dict[str, str]], @@ -188,6 +352,31 @@ def complete_action( self.last_attempts = self._action_attempts self._action_attempts = None + async def acomplete_action( + self, + messages: list[dict[str, str]], + *, + max_repair_attempts: int = 2, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict], bool] | None = None, + ) -> LLMActionResult: + self._action_attempts = [] + try: + return await super().acomplete_action( + messages, + max_repair_attempts=max_repair_attempts, + max_output_tokens=max_output_tokens, + tools=tools, + hosted_mcp_servers=hosted_mcp_servers, + mcp_approval_callback=mcp_approval_callback, + ) + finally: + if self._action_attempts is not None: + self.last_attempts = self._action_attempts + self._action_attempts = None + def stream_complete( self, messages: list[dict[str, str]], @@ -219,11 +408,33 @@ def _complete_action_response_once( ) ) + async def _acomplete_action_response_once( + self, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None = None, + tools: list[object] | None = None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict], bool] | None = None, + ) -> LLMResponse: + return await self._atry_provider_responses( + lambda provider: _acomplete_action_response_once( + provider, + messages, + max_output_tokens=max_output_tokens, + tools=tools, + hosted_mcp_servers=hosted_mcp_servers if _supports_hosted_mcp(provider) else None, + mcp_approval_callback=mcp_approval_callback if _supports_hosted_mcp(provider) else None, + ) + ) + def _try_provider_responses(self, call) -> LLMResponse: self.last_attempts = [] self.last_success_provider = None errors: list[str] = [] for provider in self.providers: + if not isinstance(provider, LLMClient): + raise TypeError("FallbackChain must be bound before use") started_at = time.monotonic() provider_name, model = _provider_identity(provider) try: @@ -231,12 +442,16 @@ def _try_provider_responses(self, call) -> LLMResponse: except Exception as exc: latency = time.monotonic() - started_at error = str(exc) + error_code, retryable, fallback_eligible = _provider_error_metadata(exc) attempt = ProviderAttempt( provider_name, model, False, latency, error=error, + error_code=error_code, + retryable=retryable, + fallback_eligible=fallback_eligible, usage=getattr(exc, "usage", None), cost=getattr(exc, "cost", None), ) @@ -244,6 +459,8 @@ def _try_provider_responses(self, call) -> LLMResponse: self.attempts.append(attempt) if self._action_attempts is not None: self._action_attempts.append(attempt) + if not isinstance(exc, LLMError) or not exc.fallback_eligible: + raise errors.append(f"{provider_name}/{model or 'unknown'}: {error}") continue latency = time.monotonic() - started_at @@ -255,7 +472,64 @@ def _try_provider_responses(self, call) -> LLMResponse: self.last_success_provider = provider return response detail = "; ".join(errors) if errors else "no providers were available" - raise LLMError(f"All fallback providers failed: {detail}") + raise LLMError( + f"All fallback providers failed: {detail}", + code="fallback_exhausted", + retryable=any(attempt.retryable is True for attempt in self.last_attempts), + ) + + async def _atry_provider_responses( + self, + call: Callable[[LLMClient], Awaitable[LLMResponse]], + ) -> LLMResponse: + self.last_attempts = [] + self.last_success_provider = None + errors: list[str] = [] + for provider in self.providers: + if not isinstance(provider, LLMClient): + raise TypeError("FallbackChain must be bound before use") + started_at = time.monotonic() + provider_name, model = _provider_identity(provider) + try: + response = await call(provider) + except Exception as exc: + latency = time.monotonic() - started_at + error = str(exc) + error_code, retryable, fallback_eligible = _provider_error_metadata(exc) + attempt = ProviderAttempt( + provider_name, + model, + False, + latency, + error=error, + error_code=error_code, + retryable=retryable, + fallback_eligible=fallback_eligible, + usage=getattr(exc, "usage", None), + cost=getattr(exc, "cost", None), + ) + self.last_attempts.append(attempt) + self.attempts.append(attempt) + if self._action_attempts is not None: + self._action_attempts.append(attempt) + if not isinstance(exc, LLMError) or not exc.fallback_eligible: + raise + errors.append(f"{provider_name}/{model or 'unknown'}: {error}") + continue + latency = time.monotonic() - started_at + attempt = ProviderAttempt(provider_name, model, True, latency, usage=response.usage, cost=response.cost) + self.last_attempts.append(attempt) + self.attempts.append(attempt) + if self._action_attempts is not None: + self._action_attempts.append(attempt) + self.last_success_provider = provider + return response + detail = "; ".join(errors) if errors else "no providers were available" + raise LLMError( + f"All fallback providers failed: {detail}", + code="fallback_exhausted", + retryable=any(attempt.retryable is True for attempt in self.last_attempts), + ) def _stream_providers( self, @@ -267,36 +541,51 @@ def _stream_providers( self.last_success_provider = None errors: list[str] = [] for provider in self.providers: + if not isinstance(provider, LLMClient): + raise TypeError("FallbackChain must be bound before use") started_at = time.monotonic() provider_name, model = _provider_identity(provider) - emitted_text = False + emitted_chunk = False usage: LLMUsage | None = None cost: LLMCost | None = None try: for chunk in _stream_complete(provider, messages, max_output_tokens=max_output_tokens): - if chunk.type == "text_delta" and chunk.text: - emitted_text = True if chunk.usage is not None: usage = chunk.usage if chunk.cost is not None: cost = chunk.cost + emitted_chunk = True yield chunk except Exception as exc: latency = time.monotonic() - started_at error = str(exc) + error_code, retryable, fallback_eligible = _provider_error_metadata(exc) attempt = ProviderAttempt( provider_name, model, False, latency, error=error, + error_code=error_code, + retryable=retryable, + fallback_eligible=fallback_eligible, usage=getattr(exc, "usage", None), cost=getattr(exc, "cost", None), ) self.last_attempts.append(attempt) self.attempts.append(attempt) - if emitted_text: - raise LLMError(f"{provider_name}/{model or 'unknown'} stream failed after emitting text: {error}") from exc + if emitted_chunk: + source_error = exc if isinstance(exc, LLMError) else None + raise LLMError( + f"{provider_name}/{model or 'unknown'} stream failed after yielding a chunk: {error}", + provider=provider_name, + model=model, + code=source_error.code if source_error is not None else "unknown", + retryable=source_error.retryable if source_error is not None else False, + fallback_eligible=False, + ) from exc + if not isinstance(exc, LLMError) or not exc.fallback_eligible: + raise errors.append(f"{provider_name}/{model or 'unknown'}: {error}") continue latency = time.monotonic() - started_at @@ -306,16 +595,26 @@ def _stream_providers( self.last_success_provider = provider return detail = "; ".join(errors) if errors else "no providers were available" - raise LLMError(f"All fallback providers failed: {detail}") + raise LLMError( + f"All fallback providers failed: {detail}", + code="fallback_exhausted", + retryable=any(attempt.retryable is True for attempt in self.last_attempts), + ) def _complete_response(provider: LLMClient, messages: list[dict[str, str]], *, max_output_tokens: int | None) -> LLMResponse: - try: - if max_output_tokens is None: - return provider.complete_response(messages) - return provider.complete_response(messages, max_output_tokens=max_output_tokens) - except TypeError: - return provider.complete_response(messages) + kwargs = {"max_output_tokens": max_output_tokens} if max_output_tokens is not None else {} + return call_with_supported_kwargs(provider.complete_response, messages, **kwargs) + + +async def _acomplete_response( + provider: LLMClient, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None, +) -> LLMResponse: + kwargs = {"max_output_tokens": max_output_tokens} if max_output_tokens is not None else {} + return await call_async_with_supported_kwargs(provider.acomplete_response, messages, **kwargs) def _stream_complete( @@ -324,13 +623,8 @@ def _stream_complete( *, max_output_tokens: int | None, ) -> Iterator[LLMStreamChunk]: - try: - if max_output_tokens is None: - yield from provider.stream_complete(messages) - return - yield from provider.stream_complete(messages, max_output_tokens=max_output_tokens) - except TypeError: - yield from provider.stream_complete(messages) + kwargs = {"max_output_tokens": max_output_tokens} if max_output_tokens is not None else {} + yield from call_with_supported_kwargs(provider.stream_complete, messages, **kwargs) def _complete_action_response_once( @@ -342,19 +636,36 @@ def _complete_action_response_once( hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, mcp_approval_callback: Callable[[dict], bool] | None = None, ) -> LLMResponse: - if max_output_tokens is None: - return provider._complete_action_response_once( - messages, - tools=tools, - hosted_mcp_servers=hosted_mcp_servers, - mcp_approval_callback=mcp_approval_callback, - ) - return provider._complete_action_response_once( + kwargs: dict[str, object] = { + "tools": tools, + "hosted_mcp_servers": hosted_mcp_servers, + "mcp_approval_callback": mcp_approval_callback, + } + if max_output_tokens is not None: + kwargs["max_output_tokens"] = max_output_tokens + return call_with_supported_kwargs(provider._complete_action_response_once, messages, **kwargs) + + +async def _acomplete_action_response_once( + provider: LLMClient, + messages: list[dict[str, str]], + *, + max_output_tokens: int | None, + tools: list[object] | None, + hosted_mcp_servers: list[object] | tuple[object, ...] | None = None, + mcp_approval_callback: Callable[[dict], bool] | None = None, +) -> LLMResponse: + kwargs: dict[str, object] = { + "tools": tools, + "hosted_mcp_servers": hosted_mcp_servers, + "mcp_approval_callback": mcp_approval_callback, + } + if max_output_tokens is not None: + kwargs["max_output_tokens"] = max_output_tokens + return await call_async_with_supported_kwargs( + provider._acomplete_action_response_once, messages, - max_output_tokens=max_output_tokens, - tools=tools, - hosted_mcp_servers=hosted_mcp_servers, - mcp_approval_callback=mcp_approval_callback, + **kwargs, ) @@ -369,11 +680,23 @@ def _provider_identity(provider: object) -> tuple[str, str | None]: return str(provider_name), str(model) if model is not None else None +def _provider_error_metadata(exc: Exception) -> tuple[str | None, bool | None, bool | None]: + if not isinstance(exc, LLMError): + return None, None, None + return exc.code, exc.retryable, exc.fallback_eligible + + __all__ = [ + "AnthropicProvider", + "BedrockProvider", + "BindableLLM", "DeepSeekProvider", "FallbackChain", "FallbackStrategy", + "GeminiProvider", "LocalProvider", + "OpenAICompatibleProvider", "OpenAIProvider", + "OpenRouterProvider", "ProviderAttempt", ] diff --git a/chulk/llm/usage.py b/chulk/llm/usage.py index f6cf0c9..84e2666 100644 --- a/chulk/llm/usage.py +++ b/chulk/llm/usage.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from decimal import Decimal -from typing import Any +from typing import Any, cast @dataclass(frozen=True) @@ -120,8 +120,8 @@ def normalize_openai_usage(usage: object) -> LLMUsage | None: ) -def normalize_deepseek_usage(usage: object) -> LLMUsage | None: - """Return normalized usage from a DeepSeek chat-completions usage object.""" +def normalize_chat_completions_usage(usage: object) -> LLMUsage | None: + """Return normalized usage from an OpenAI-compatible chat-completions response.""" if usage is None: return None prompt_tokens = _int_value(_value(usage, "prompt_tokens")) @@ -152,6 +152,11 @@ def normalize_deepseek_usage(usage: object) -> LLMUsage | None: ) +def normalize_deepseek_usage(usage: object) -> LLMUsage | None: + """Return normalized usage from a DeepSeek chat-completions usage object.""" + return normalize_chat_completions_usage(usage) + + def estimate_usage(messages: list[dict[str, str]], content: str) -> LLMUsage: """Estimate usage with Chulk's deterministic context estimator.""" from chulk.core.context import estimate_message_tokens, estimate_tokens @@ -210,6 +215,7 @@ def aggregate_cost(costs: list[LLMCost | None]) -> LLMCost | None: def usage_from_dict(payload: object) -> LLMUsage | None: if not isinstance(payload, dict): return None + raw = payload.get("raw") return LLMUsage( input_tokens=_int_value(payload.get("input_tokens")), output_tokens=_int_value(payload.get("output_tokens")), @@ -221,7 +227,7 @@ def usage_from_dict(payload: object) -> LLMUsage | None: estimated=bool(payload.get("estimated")), cache_split_estimated=bool(payload.get("cache_split_estimated")), source=str(payload.get("source") or "provider"), - raw=payload.get("raw") if isinstance(payload.get("raw"), dict) else {}, + raw=cast(dict[str, Any], raw) if isinstance(raw, dict) else {}, ) @@ -276,7 +282,7 @@ def _int_value(value: object) -> int: if value is None: return 0 try: - return max(0, int(value)) + return max(0, int(cast(Any, value))) except (TypeError, ValueError): return 0 diff --git a/chulk/main.py b/chulk/main.py index c032523..04957b8 100644 --- a/chulk/main.py +++ b/chulk/main.py @@ -6,6 +6,7 @@ from collections.abc import Sequence import sys from typing import Callable +from urllib.parse import urlsplit, urlunsplit from chulk import __version__ from chulk.cli import ( @@ -31,13 +32,19 @@ from chulk.config import Config, load_cli_config from chulk.core import Agent from chulk.llm import ( + AnthropicProvider, + BedrockProvider, + BindableLLM, DeepSeekProvider, FallbackChain, + GeminiProvider, LLMClient, LLMConfigurationError, LLMError, LocalProvider, + OpenAICompatibleProvider, OpenAIProvider, + OpenRouterProvider, ) from chulk.presets import software_engineer from chulk.runtime import create_agent @@ -62,9 +69,19 @@ def format_config(config: Config) -> str: "permission_profile": config.permission_profile, "openai_api_key": "set" if config.openai_api_key else "not set", "deepseek_api_key": "set" if config.deepseek_api_key else "not set", - "deepseek_base_url": config.deepseek_base_url, + "deepseek_base_url": _format_base_url(config.deepseek_base_url), "local_api_key": "set" if config.local_api_key else "not set", - "local_base_url": config.local_base_url, + "local_base_url": _format_base_url(config.local_base_url), + "openai_compatible_api_key": "set" if config.openai_compatible_api_key else "not set", + "openai_compatible_base_url": _format_base_url(config.openai_compatible_base_url), + "openrouter_api_key": "set" if config.openrouter_api_key else "not set", + "openrouter_base_url": _format_base_url(config.openrouter_base_url), + "anthropic_api_key": "set" if config.anthropic_api_key else "not set", + "anthropic_base_url": _format_base_url(config.anthropic_base_url), + "bedrock_api_key": "set" if config.bedrock_api_key else "not set", + "bedrock_base_url": _format_base_url(config.bedrock_base_url), + "gemini_api_key": "set" if config.gemini_api_key else "not set", + "gemini_base_url": _format_base_url(config.gemini_base_url), "history_limit": config.history_limit, "max_tool_calls_per_turn": config.max_tool_calls_per_turn, "max_skills_per_turn": config.max_skills_per_turn, @@ -83,6 +100,26 @@ def format_config(config: Config) -> str: return "\n".join(lines) +def _format_base_url(value: str | None) -> str: + """Render a base URL without exposing credentials or URL parameters.""" + if value is None or not value.strip(): + return "not set" + clean_value = value.strip() + if any(ord(character) < 32 or ord(character) == 127 for character in clean_value): + return "set (value hidden)" + try: + parsed = urlsplit(clean_value) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + return "set (value hidden)" + hostname = parsed.hostname + if ":" in hostname and not hostname.startswith("["): + hostname = f"[{hostname}]" + port = f":{parsed.port}" if parsed.port is not None else "" + except ValueError: + return "set (value hidden)" + return urlunsplit((parsed.scheme, f"{hostname}{port}", parsed.path, "", "")) + + def create_cli_agent( config: Config, llm_client_factory: Callable[[Config], LLMClient] | None = None, @@ -112,7 +149,7 @@ def create_cli_agent( def create_cli_llm(config: Config) -> FallbackChain: """Create the CLI LLM chain from public provider objects.""" - providers: list[OpenAIProvider | DeepSeekProvider | LocalProvider] = [ + providers: list[LLMClient | BindableLLM] = [ _create_provider_spec(config.llm_provider, config.model) ] providers.extend( @@ -122,13 +159,35 @@ def create_cli_llm(config: Config) -> FallbackChain: return FallbackChain(providers=providers) -def _create_provider_spec(provider: str, model: str) -> OpenAIProvider | DeepSeekProvider | LocalProvider: +def _create_provider_spec( + provider: str, + model: str, +) -> ( + OpenAIProvider + | DeepSeekProvider + | LocalProvider + | OpenAICompatibleProvider + | OpenRouterProvider + | AnthropicProvider + | BedrockProvider + | GeminiProvider +): if provider == "openai": return OpenAIProvider(model=model) if provider == "deepseek": return DeepSeekProvider(model=model) if provider == "local": return LocalProvider(model=model) + if provider == "openai-compatible": + return OpenAICompatibleProvider(model=model) + if provider == "openrouter": + return OpenRouterProvider(model=model) + if provider == "anthropic": + return AnthropicProvider(model=model) + if provider == "bedrock": + return BedrockProvider(model=model) + if provider == "gemini": + return GeminiProvider(model=model) raise LLMConfigurationError(f"Unsupported CLI LLM provider: {provider}") diff --git a/chulk/mcp/bridge.py b/chulk/mcp/bridge.py index e990e81..bab564b 100644 --- a/chulk/mcp/bridge.py +++ b/chulk/mcp/bridge.py @@ -54,7 +54,12 @@ async def _list_tools(self) -> list[object]: async with self._connect() as session: await session.initialize() result = await session.list_tools() - return list(_value(result, "tools") or []) + tools = _value(result, "tools") + if tools is None: + return [] + if not isinstance(tools, Iterable): + raise TypeError("MCP list_tools response did not contain an iterable tools value") + return list(tools) async def _call_tool(self, name: str, arguments: dict[str, Any]) -> object: async with self._connect() as session: @@ -81,7 +86,7 @@ class _StreamableHttpSessionContext: def __init__(self, client_context, session_type) -> None: self.client_context = client_context self.session_type = session_type - self.session = None + self.session: Any | None = None async def __aenter__(self): read_stream, write_stream, _ = await self.client_context.__aenter__() @@ -89,8 +94,11 @@ async def __aenter__(self): return await self.session.__aenter__() async def __aexit__(self, exc_type, exc, tb): + session = self.session + if session is None: + raise RuntimeError("MCP session context exited before it was entered") try: - return await self.session.__aexit__(exc_type, exc, tb) + return await session.__aexit__(exc_type, exc, tb) finally: await self.client_context.__aexit__(exc_type, exc, tb) diff --git a/chulk/memory/__init__.py b/chulk/memory/__init__.py index db950ea..f4e61f8 100644 --- a/chulk/memory/__init__.py +++ b/chulk/memory/__init__.py @@ -4,6 +4,7 @@ from chulk.memory.models import MemoryExtractionCandidate, MemoryProposalRecord, MemoryRecord from chulk.memory.policy import MemoryPolicy, MemoryPolicyResult from chulk.memory.retrieval import text_to_embedding +from chulk.memory.security import MemorySecretError from chulk.memory.sqlite_store import SQLiteMemoryStore, select_memories_for_prompt from chulk.memory.store import ConversationMemory, Memory, new_memory @@ -15,6 +16,7 @@ "MemoryPolicyResult", "MemoryProposalRecord", "MemoryRecord", + "MemorySecretError", "SQLiteMemoryStore", "extract_memory_candidates", "new_memory", diff --git a/chulk/memory/policy.py b/chulk/memory/policy.py index 5ade3e9..5b2e8f6 100644 --- a/chulk/memory/policy.py +++ b/chulk/memory/policy.py @@ -6,6 +6,7 @@ from chulk.capabilities import MemoryMode from chulk.memory.models import MemoryExtractionCandidate, MemoryProposalRecord +from chulk.memory.security import MemorySecretError, ensure_memory_payload_safe from chulk.memory.sqlite_store import SQLiteMemoryStore @@ -37,33 +38,42 @@ def handle_candidates( if self.mode in {MemoryMode.OFF, MemoryMode.READ_ONLY}: return MemoryPolicyResult() if self.mode is MemoryMode.MANUAL: - proposals = tuple( - self.store.create_memory_proposal( - candidate.content, - tags=candidate.tags, - metadata=candidate.metadata, - importance=candidate.importance, - source=candidate.source, - confidence=candidate.confidence, - evidence=evidence, - conversation_id=conversation_id, - turn_id=turn_id, + proposal_ids = [] + for candidate in candidates: + try: + proposal_ids.append( + self.store.create_memory_proposal( + candidate.content, + tags=candidate.tags, + metadata=candidate.metadata, + importance=candidate.importance, + source=candidate.source, + confidence=candidate.confidence, + evidence=evidence, + conversation_id=conversation_id, + turn_id=turn_id, + ) + ) + except MemorySecretError: + continue + return MemoryPolicyResult(proposal_ids=tuple(proposal_ids)) + + memory_ids = [] + for candidate in candidates: + try: + memory_ids.append( + self.store.save_memory( + candidate.content, + tags=candidate.tags, + metadata=candidate.metadata, + importance=candidate.importance, + source=candidate.source, + confidence=candidate.confidence, + ) ) - for candidate in candidates - ) - return MemoryPolicyResult(proposal_ids=proposals) - memory_ids = tuple( - self.store.save_memory( - candidate.content, - tags=candidate.tags, - metadata=candidate.metadata, - importance=candidate.importance, - source=candidate.source, - confidence=candidate.confidence, - ) - for candidate in candidates - ) - return MemoryPolicyResult(accepted_memory_ids=memory_ids) + except MemorySecretError: + continue + return MemoryPolicyResult(accepted_memory_ids=tuple(memory_ids)) def propose_explicit( self, @@ -77,6 +87,7 @@ def propose_explicit( conversation_id: str | None = None, turn_id: str | None = None, ) -> MemoryPolicyResult: + ensure_memory_payload_safe(content=content, tags=tags or [], metadata=metadata or {}, source=source) candidate = MemoryExtractionCandidate( content=content, tags=tags or [], diff --git a/chulk/memory/security.py b/chulk/memory/security.py new file mode 100644 index 0000000..da2ecb0 --- /dev/null +++ b/chulk/memory/security.py @@ -0,0 +1,218 @@ +"""Credential detection for durable memory writes.""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +import re +from typing import Any + + +class MemorySecretError(ValueError): + """Raised when a durable memory payload contains credential-like data.""" + + +_REJECTION_MESSAGE = "Credential-like data is not allowed in durable memory." +_SECRET_FIELD_RE = re.compile( + r"(?ix)^(?:" + r"api[_-]?key|api[_-]?keys|access[_-]?key|access[_-]?token|refresh[_-]?token|" + r"auth[_-]?token|authorization|bearer[_-]?token|client[_-]?secret|cookie|credentials?|" + r"password|passwd|pwd|private[_-]?key|secret|secret[_-]?key(?:[_-]?base)?|token|" + r"[a-z0-9]+(?:[_-][a-z0-9]+)*[_-](?:api[_-]?key|access[_-]?key|access[_-]?token|" + r"refresh[_-]?token|auth[_-]?token|client[_-]?secret|credential|password|passwd|pwd|" + r"private[_-]?key|secret|secret[_-]?key(?:[_-]?base)?|token)" + r")$" +) +_SECRET_LABEL = ( + r"(?:(?:[a-z0-9]+[_-])*(?:api[_ -]?key|access[_ -]?key|access[_ -]?token|" + r"refresh[_ -]?token|auth[_ -]?token|authorization|bearer[_ -]?token|" + r"client[_ -]?secret|cookie|credential|password|passwd|pwd|private[_ -]?key|" + r"secret|secret[_ -]?key(?:[_ -]?base)?|token)|[a-z0-9]*(?:apikey|accesskey|" + r"accesstoken|refreshtoken|authtoken|clientsecret|privatekey|secretkeybase))" +) +_ASSIGNMENT_RE = re.compile( + rf"(?ix)\b(?P