From 66be02f88483902548b663486b9b681f184e1b14 Mon Sep 17 00:00:00 2001 From: Prodman Devokadev Date: Mon, 31 Aug 2026 00:45:03 +0530 Subject: [PATCH 1/3] feat: add LiteLLM as an optional OpenAI-compatible provider --- CHANGELOG.md | 3 + crates/webclaw-cli/src/main.rs | 15 +++- crates/webclaw-llm/src/chain.rs | 14 +++- crates/webclaw-llm/src/providers/litellm.rs | 83 +++++++++++++++++++++ crates/webclaw-llm/src/providers/mod.rs | 1 + env.example | 5 ++ 6 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 crates/webclaw-llm/src/providers/litellm.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7004604..ffba8e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ Format follows [Keep a Changelog](https://keepachangelog.com/). ## [Unreleased] +### Added +- **Optional LiteLLM gateway provider.** Extraction and summarization can route through a self-hosted LiteLLM proxy (OpenAI-compatible), reaching many upstream models behind one endpoint. Opt-in via `LITELLM_API_KEY`; it is added last in the provider chain, so it never changes provider priority for existing users. + ## [0.6.22] - 2026-08-30 ### Added diff --git a/crates/webclaw-cli/src/main.rs b/crates/webclaw-cli/src/main.rs index 6f79c2f..b6b99c2 100644 --- a/crates/webclaw-cli/src/main.rs +++ b/crates/webclaw-cli/src/main.rs @@ -339,7 +339,7 @@ struct Cli { #[arg(long, num_args = 0..=1, default_missing_value = "3")] summarize: Option, - /// Force a specific LLM provider (ollama, openai, atlascloud, anthropic, orcarouter) + /// Force a specific LLM provider (ollama, openai, atlascloud, anthropic, orcarouter, litellm) #[arg(long, env = "WEBCLAW_LLM_PROVIDER")] llm_provider: Option, @@ -2284,15 +2284,24 @@ async fn build_llm_provider(cli: &Cli) -> Result, String> { .ok_or("ANTHROPIC_API_KEY not set")?; Ok(Box::new(provider)) } + "litellm" => { + let provider = webclaw_llm::providers::litellm::LiteLlmProvider::new( + None, + cli.llm_base_url.clone(), + cli.llm_model.clone(), + ) + .ok_or("LITELLM_API_KEY not set")?; + Ok(Box::new(provider)) + } other => Err(format!( - "unknown LLM provider: {other} (use ollama, openai, atlascloud, anthropic, or orcarouter)" + "unknown LLM provider: {other} (use ollama, openai, atlascloud, anthropic, orcarouter, or litellm)" )), } } else { let chain = webclaw_llm::ProviderChain::default().await; if chain.is_empty() { return Err( - "no LLM providers available -- start Ollama or set OPENAI_API_KEY / ANTHROPIC_API_KEY / ORCAROUTER_API_KEY" + "no LLM providers available -- start Ollama or set OPENAI_API_KEY / ANTHROPIC_API_KEY / ORCAROUTER_API_KEY / LITELLM_API_KEY" .into(), ); } diff --git a/crates/webclaw-llm/src/chain.rs b/crates/webclaw-llm/src/chain.rs index 2084784..2a1ff40 100644 --- a/crates/webclaw-llm/src/chain.rs +++ b/crates/webclaw-llm/src/chain.rs @@ -8,7 +8,8 @@ use crate::error::LlmError; use crate::provider::{CompletionRequest, LlmProvider}; use crate::providers::{ anthropic::AnthropicProvider, atlascloud::AtlasCloudProvider, gemini::GeminiProvider, - ollama::OllamaProvider, openai::OpenAiProvider, orcarouter::OrcaRouterProvider, + litellm::LiteLlmProvider, ollama::OllamaProvider, openai::OpenAiProvider, + orcarouter::OrcaRouterProvider, }; pub struct ProviderChain { @@ -22,8 +23,10 @@ impl ProviderChain { /// Gemini sits ahead of Anthropic so Google Cloud credits are preferred, /// with Anthropic as the last-resort fallback. Atlas Cloud is opt-in and /// added last (only when `ATLASCLOUD_API_KEY` is set), so it never preempts - /// an already-configured provider. OrcaRouter is also opt-in and added last, - /// only when `ORCAROUTER_API_KEY` is set. + /// an already-configured provider. OrcaRouter and LiteLLM are also opt-in + /// and added last, only when `ORCAROUTER_API_KEY` / `LITELLM_API_KEY` is + /// set. A LiteLLM proxy is OpenAI-compatible, so it reaches 100+ upstream + /// providers through one endpoint. pub async fn default() -> Self { Self::build_default(true).await } @@ -73,6 +76,11 @@ impl ProviderChain { providers.push(Box::new(orcarouter)); } + if let Some(litellm) = LiteLlmProvider::new(None, None, None) { + debug!("litellm configured, adding to chain"); + providers.push(Box::new(litellm)); + } + Self { providers } } diff --git a/crates/webclaw-llm/src/providers/litellm.rs b/crates/webclaw-llm/src/providers/litellm.rs new file mode 100644 index 0000000..f7c5cd0 --- /dev/null +++ b/crates/webclaw-llm/src/providers/litellm.rs @@ -0,0 +1,83 @@ +/// LiteLLM provider — OpenAI-compatible chat completions against a LiteLLM proxy. +/// +/// A LiteLLM proxy speaks the OpenAI wire format, so this provider reuses the +/// `OpenAiProvider` transport unchanged. Pointing it at a LiteLLM gateway lets +/// webclaw reach 100+ upstream providers (OpenAI, Anthropic, Bedrock, Vertex +/// AI, Azure, and more) through a single endpoint with centralized keys. +use async_trait::async_trait; + +use crate::error::LlmError; +use crate::provider::{CompletionRequest, LlmProvider}; + +use super::openai::OpenAiProvider; + +pub struct LiteLlmProvider { + inner: OpenAiProvider, +} + +impl LiteLlmProvider { + /// Returns `None` if no LiteLLM API key is available (param or env). + pub fn new( + key_override: Option, + base_url: Option, + model: Option, + ) -> Option { + let key = super::load_api_key(key_override, "LITELLM_API_KEY")?; + let base_url = base_url + .or_else(|| std::env::var("LITELLM_BASE_URL").ok()) + .unwrap_or_else(|| "http://localhost:4000/v1".into()); + let model = model + .or_else(|| std::env::var("LITELLM_MODEL").ok()) + .unwrap_or_else(|| "gpt-4o-mini".into()); + let inner = OpenAiProvider::new(Some(key), Some(base_url), Some(model))?; + Some(Self { inner }) + } + + pub fn default_model(&self) -> &str { + self.inner.default_model() + } +} + +#[async_trait] +impl LlmProvider for LiteLlmProvider { + async fn complete(&self, request: &CompletionRequest) -> Result { + self.inner.complete(request).await + } + + async fn is_available(&self) -> bool { + self.inner.is_available().await + } + + fn name(&self) -> &str { + "litellm" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_key_returns_none() { + assert!(LiteLlmProvider::new(Some(String::new()), None, None).is_none()); + } + + #[test] + fn explicit_key_constructs_with_litellm_defaults() { + let provider = + LiteLlmProvider::new(Some("test-key".into()), None, None).expect("should construct"); + assert_eq!(provider.name(), "litellm"); + assert_eq!(provider.default_model(), "gpt-4o-mini"); + } + + #[test] + fn explicit_model_override() { + let provider = LiteLlmProvider::new( + Some("test-key".into()), + Some("http://proxy.example.com:4000/v1".into()), + Some("claude-sonnet-4-6".into()), + ) + .expect("should construct"); + assert_eq!(provider.default_model(), "claude-sonnet-4-6"); + } +} diff --git a/crates/webclaw-llm/src/providers/mod.rs b/crates/webclaw-llm/src/providers/mod.rs index 6421922..559e17b 100644 --- a/crates/webclaw-llm/src/providers/mod.rs +++ b/crates/webclaw-llm/src/providers/mod.rs @@ -1,6 +1,7 @@ pub mod anthropic; pub mod atlascloud; pub mod gemini; +pub mod litellm; pub mod ollama; pub mod openai; pub mod orcarouter; diff --git a/env.example b/env.example index 7039280..b76cf9c 100644 --- a/env.example +++ b/env.example @@ -23,6 +23,11 @@ OLLAMA_MODEL=qwen3:8b # ORCAROUTER_BASE_URL — defaults to https://api.orcarouter.ai/v1 # ORCAROUTER_MODEL — defaults to orcarouter/auto +# LiteLLM proxy (optional OpenAI-compatible gateway to 100+ providers) +# LITELLM_API_KEY — set your LiteLLM proxy key +# LITELLM_BASE_URL — defaults to http://localhost:4000/v1 +# LITELLM_MODEL — defaults to gpt-4o-mini + # --- Proxy --- # Single proxy From 397c361d02216eca17199dd35004f2591ada1588 Mon Sep 17 00:00:00 2001 From: Prodman Devokadev Date: Mon, 31 Aug 2026 01:17:13 +0530 Subject: [PATCH 2/3] docs: mention LiteLLM in provider-chain comment and README env table --- README.md | 2 ++ crates/webclaw-llm/src/chain.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2cf0b1e..3aedd7c 100644 --- a/README.md +++ b/README.md @@ -371,6 +371,8 @@ webclaw/ | `ANTHROPIC_BASE_URL` | Anthropic-compatible base URL | | `ORCAROUTER_API_KEY` | OrcaRouter LLM provider key | | `ORCAROUTER_BASE_URL` | OrcaRouter base URL (defaults to https://api.orcarouter.ai/v1) | +| `LITELLM_API_KEY` | LiteLLM proxy key (OpenAI-compatible gateway) | +| `LITELLM_BASE_URL` | LiteLLM proxy base URL (defaults to http://localhost:4000/v1) | | `WEBCLAW_PROXY` | Single proxy URL | | `WEBCLAW_PROXY_FILE` | Proxy pool file | diff --git a/crates/webclaw-llm/src/chain.rs b/crates/webclaw-llm/src/chain.rs index 2a1ff40..2b0eb83 100644 --- a/crates/webclaw-llm/src/chain.rs +++ b/crates/webclaw-llm/src/chain.rs @@ -17,7 +17,7 @@ pub struct ProviderChain { } impl ProviderChain { - /// Build the default chain: Ollama -> OpenAI -> Gemini -> Anthropic -> Atlas Cloud -> OrcaRouter. + /// Build the default chain: Ollama -> OpenAI -> Gemini -> Anthropic -> Atlas Cloud -> OrcaRouter -> LiteLLM. /// Ollama is always added (availability checked at call time). /// Cloud providers are only added if their API keys are configured. /// Gemini sits ahead of Anthropic so Google Cloud credits are preferred, From 345422db95377c8ee2a3ad46392419fd3c7e4d73 Mon Sep 17 00:00:00 2001 From: Prodman Devokadev Date: Mon, 31 Aug 2026 22:25:13 +0530 Subject: [PATCH 3/3] fix: address review - ignore env-dependent default-model test, document LITELLM_MODEL, generic changelog wording --- CHANGELOG.md | 2 +- README.md | 1 + crates/webclaw-llm/src/providers/litellm.rs | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffba8e4..2884567 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Format follows [Keep a Changelog](https://keepachangelog.com/). ## [Unreleased] ### Added -- **Optional LiteLLM gateway provider.** Extraction and summarization can route through a self-hosted LiteLLM proxy (OpenAI-compatible), reaching many upstream models behind one endpoint. Opt-in via `LITELLM_API_KEY`; it is added last in the provider chain, so it never changes provider priority for existing users. +- **LiteLLM support for AI features.** Point webclaw at your own LiteLLM setup to use many different models for extraction and summarization through a single connection. Enable it by setting `LITELLM_API_KEY`; existing model choices are unaffected. ## [0.6.22] - 2026-08-30 diff --git a/README.md b/README.md index 3aedd7c..932096e 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,7 @@ webclaw/ | `ORCAROUTER_BASE_URL` | OrcaRouter base URL (defaults to https://api.orcarouter.ai/v1) | | `LITELLM_API_KEY` | LiteLLM proxy key (OpenAI-compatible gateway) | | `LITELLM_BASE_URL` | LiteLLM proxy base URL (defaults to http://localhost:4000/v1) | +| `LITELLM_MODEL` | LiteLLM default model (defaults to gpt-4o-mini) | | `WEBCLAW_PROXY` | Single proxy URL | | `WEBCLAW_PROXY_FILE` | Proxy pool file | diff --git a/crates/webclaw-llm/src/providers/litellm.rs b/crates/webclaw-llm/src/providers/litellm.rs index f7c5cd0..7ed48bc 100644 --- a/crates/webclaw-llm/src/providers/litellm.rs +++ b/crates/webclaw-llm/src/providers/litellm.rs @@ -63,6 +63,7 @@ mod tests { } #[test] + #[ignore = "reads LITELLM_MODEL from the process env; run with --test-threads=1"] fn explicit_key_constructs_with_litellm_defaults() { let provider = LiteLlmProvider::new(Some("test-key".into()), None, None).expect("should construct");