From e85a95eb4401e99f491947fd7af116d2405cf640 Mon Sep 17 00:00:00 2001 From: RheagalFire Date: Wed, 8 Jul 2026 16:15:37 +0530 Subject: [PATCH 1/2] feat: add LiteLLM as AI gateway provider --- src/providers/registry.ts | 3 ++- src/utils/config.ts | 13 +++++++++++-- src/utils/provider-models.ts | 2 ++ src/web/api/providers.ts | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 2de70f09..3978871f 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -18,7 +18,7 @@ async function createProvider(pc: ProviderConfig): Promise { // this entirely. const { OpenAICompatProvider } = await import('./openai-compat.js'); return new OpenAICompatProvider(pc, { useChatApi: true }); - } else if (pc.name === 'ollamaCloud' || pc.name === 'openaiCompat') { + } else if (pc.name === 'ollamaCloud' || pc.name === 'openaiCompat' || pc.name === 'litellm') { const { OpenAICompatProvider } = await import('./openai-compat.js'); return new OpenAICompatProvider(pc, { useChatApi: true }); } else if (pc.name === 'mimo' || pc.name === 'mimoTokenPlan') { @@ -60,6 +60,7 @@ export class ProviderRegistry { config.providers.mimoTokenPlan, config.providers.chatgptWeb, config.providers.githubCopilot, + config.providers.litellm, ]; // Load only configured providers in parallel diff --git a/src/utils/config.ts b/src/utils/config.ts index f7ce2b49..d82468d0 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -58,7 +58,8 @@ export type ProviderName = | 'mimo' | 'mimoTokenPlan' | 'chatgptWeb' - | 'githubCopilot'; + | 'githubCopilot' + | 'litellm'; export interface MercuryConfig { identity: { @@ -79,6 +80,7 @@ export interface MercuryConfig { mimoTokenPlan: ProviderConfig; chatgptWeb: ProviderConfig; githubCopilot: ProviderConfig; + litellm: ProviderConfig; }; channels: { telegram: { @@ -281,6 +283,13 @@ export function getDefaultConfig(): MercuryConfig { model: getEnv('GITHUB_COPILOT_MODEL', 'gpt-4o'), enabled: getEnvBool('GITHUB_COPILOT_ENABLED', false), }, + litellm: { + name: 'litellm', + apiKey: getEnv('LITELLM_API_KEY', ''), + baseUrl: getEnv('LITELLM_BASE_URL', 'http://localhost:4000/v1'), + model: getEnv('LITELLM_MODEL', 'gpt-4o-mini'), + enabled: getEnvBool('LITELLM_ENABLED', false), + }, }, channels: { telegram: { @@ -452,7 +461,7 @@ export function isProviderConfigured(provider: ProviderConfig): boolean { if (provider.name === 'ollamaCloud') { return provider.apiKey.length > 0 && provider.baseUrl.length > 0; } - if (provider.name === 'openaiCompat') { + if (provider.name === 'openaiCompat' || provider.name === 'litellm') { return provider.baseUrl.length > 0 && provider.model.length > 0; } if (provider.name === 'chatgptWeb') { diff --git a/src/utils/provider-models.ts b/src/utils/provider-models.ts index 3a50d38f..7e5d9871 100644 --- a/src/utils/provider-models.ts +++ b/src/utils/provider-models.ts @@ -200,6 +200,7 @@ function chooseRecommendedModel( mimoTokenPlan: MIMO_TOKEN_PLAN_PREFERRED_MODELS, chatgptWeb: CHATGPT_WEB_PREFERRED_MODELS, githubCopilot: GITHUB_COPILOT_PREFERRED_MODELS, + litellm: OPENAI_COMPAT_PREFERRED_MODELS, }; for (const candidate of preferredByProvider[provider]) { @@ -238,6 +239,7 @@ export function buildModelCatalog( mimoTokenPlan: MIMO_TOKEN_PLAN_PREFERRED_MODELS, chatgptWeb: CHATGPT_WEB_PREFERRED_MODELS, githubCopilot: GITHUB_COPILOT_PREFERRED_MODELS, + litellm: OPENAI_COMPAT_PREFERRED_MODELS, }; const withoutRecommended = filtered.filter((model) => model !== recommendedModel); diff --git a/src/web/api/providers.ts b/src/web/api/providers.ts index 8a9d3006..c81d7033 100644 --- a/src/web/api/providers.ts +++ b/src/web/api/providers.ts @@ -26,7 +26,7 @@ providers.post('/api/providers/:name', async (c) => { const body = await c.req.json(); const config = loadConfig(); - const validNames: ProviderName[] = ['openai', 'anthropic', 'deepseek', 'grok', 'ollamaCloud', 'ollamaLocal']; + const validNames: ProviderName[] = ['openai', 'anthropic', 'deepseek', 'grok', 'ollamaCloud', 'ollamaLocal', 'litellm']; if (!validNames.includes(providerName)) { return c.json({ error: 'Unknown provider' }, 400); } From 797980a144ed16715d4d2075f20258d6029fda2b Mon Sep 17 00:00:00 2001 From: RheagalFire Date: Wed, 8 Jul 2026 20:51:56 +0530 Subject: [PATCH 2/2] fix: handle keyless litellm proxy and improve error messages --- src/providers/openai-compat.ts | 72 ++++++++++++++++++++++++---------- src/utils/provider-models.ts | 6 ++- src/web/api/providers.ts | 11 +++++- 3 files changed, 65 insertions(+), 24 deletions(-) diff --git a/src/providers/openai-compat.ts b/src/providers/openai-compat.ts index 225a1c7f..6ffa3d0a 100644 --- a/src/providers/openai-compat.ts +++ b/src/providers/openai-compat.ts @@ -25,37 +25,67 @@ export class OpenAICompatProvider extends BaseProvider { : this.client(config.model); } + private formatApiError(err: unknown): Error { + if (err instanceof Error) { + const msg = err.message || ''; + // Surface common HTTP error codes with actionable messages + if (msg.includes('401') || msg.toLowerCase().includes('unauthorized')) { + return new Error(`[${this.name}] Authentication failed - check your API key or proxy credentials.`); + } + if (msg.includes('404') || msg.toLowerCase().includes('not found')) { + return new Error(`[${this.name}] Model "${this.model}" not found - verify the model name and base URL.`); + } + if (msg.includes('429') || msg.toLowerCase().includes('rate limit') || msg.toLowerCase().includes('too many')) { + return new Error(`[${this.name}] Rate limited - wait a moment before retrying.`); + } + return new Error(`[${this.name}] ${msg}`); + } + return new Error(`[${this.name}] Unknown error during API call.`); + } + async generateText(prompt: string, systemPrompt: string): Promise { - const result = await generateText({ - model: this.modelInstance, - system: systemPrompt, - prompt, - }); + try { + const result = await generateText({ + model: this.modelInstance, + system: systemPrompt, + prompt, + }); - return { - text: result.text, - inputTokens: result.usage?.inputTokens ?? 0, - outputTokens: result.usage?.outputTokens ?? 0, - totalTokens: (result.usage?.inputTokens ?? 0) + (result.usage?.outputTokens ?? 0), - model: this.model, - provider: this.name, - }; + return { + text: result.text, + inputTokens: result.usage?.inputTokens ?? 0, + outputTokens: result.usage?.outputTokens ?? 0, + totalTokens: (result.usage?.inputTokens ?? 0) + (result.usage?.outputTokens ?? 0), + model: this.model, + provider: this.name, + }; + } catch (err) { + throw this.formatApiError(err); + } } async *streamText(prompt: string, systemPrompt: string): AsyncIterable { - const result = streamText({ - model: this.modelInstance, - system: systemPrompt, - prompt, - }); + try { + const result = streamText({ + model: this.modelInstance, + system: systemPrompt, + prompt, + }); - for await (const chunk of (await result).textStream) { - yield { text: chunk, done: false }; + for await (const chunk of (await result).textStream) { + yield { text: chunk, done: false }; + } + yield { text: '', done: true }; + } catch (err) { + throw this.formatApiError(err); } - yield { text: '', done: true }; } isAvailable(): boolean { + // LiteLLM proxies and local Ollama can run without an API key + if (this.config.name === 'litellm' || this.config.name === 'ollamaLocal') { + return this.config.baseUrl.length > 0; + } return this.config.apiKey.length > 0; } diff --git a/src/utils/provider-models.ts b/src/utils/provider-models.ts index 7e5d9871..1e60242d 100644 --- a/src/utils/provider-models.ts +++ b/src/utils/provider-models.ts @@ -264,6 +264,8 @@ async function fetchOpenAICompatModels(provider: ProviderName, config: ProviderC errorMessage = 'Mercury could not fetch models for this DeepSeek key. Please re-enter it.'; } else if (provider === 'openaiCompat') { errorMessage = 'Mercury could not fetch models from this server. Please check the base URL and try again.'; + } else if (provider === 'litellm') { + errorMessage = 'Mercury could not fetch models from the LiteLLM proxy. Please check the base URL and ensure the proxy is running.'; } else { errorMessage = 'Mercury could not fetch models for this OpenAI key. Please re-enter it.'; } @@ -280,7 +282,7 @@ async function fetchOpenAICompatModels(provider: ProviderName, config: ProviderC if (provider === 'deepseek') { return id.startsWith('deepseek-'); } - if (provider === 'openaiCompat') { + if (provider === 'openaiCompat' || provider === 'litellm') { return id.length > 0; } return isOpenAIChatModel(id); @@ -421,7 +423,7 @@ export async function fetchProviderModelCatalog( return fetchOllamaLocalModels(config); } - if (provider === 'openaiCompat') { + if (provider === 'openaiCompat' || provider === 'litellm') { return fetchOpenAICompatModels(provider, config); } diff --git a/src/web/api/providers.ts b/src/web/api/providers.ts index c81d7033..c3444526 100644 --- a/src/web/api/providers.ts +++ b/src/web/api/providers.ts @@ -46,9 +46,18 @@ providers.post('/api/providers/:name/test', async (c) => { const config = loadConfig(); const p = config.providers[providerName]; - if (!p || !p.apiKey) { + if (!p) { + return c.json({ error: 'Provider not found' }, 400); + } + + // LiteLLM proxies can be keyless - only require baseUrl + const needsKey = providerName !== 'litellm' && providerName !== 'ollamaLocal'; + if (needsKey && !p.apiKey) { return c.json({ error: 'No API key configured' }, 400); } + if (!needsKey && !p.baseUrl) { + return c.json({ error: 'No base URL configured' }, 400); + } try { const { fetchProviderModelCatalog } = await import('../../utils/provider-models.js');