diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index 812b4f9..bb48699 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -50,41 +50,7 @@ class GenerateAgentDescriptionRequest(BaseModel): ) -def _get_llm_provider_and_model( - organization_id: UUID, - db: Session, - provider: Optional[str] = None, - model: Optional[str] = None, -): - """Resolve the LLM provider and model to use, falling back to org defaults.""" - from app.models.database import AIProvider - from app.models.enums import ModelProvider - - if provider and model: - try: - provider_enum = ModelProvider(provider.lower()) - except ValueError: - raise HTTPException(400, f"Unsupported LLM provider: {provider}") - return provider_enum, model - - for prov in [ModelProvider.OPENAI, ModelProvider.ANTHROPIC, ModelProvider.GOOGLE]: - ai_prov = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - AIProvider.is_active == True, - AIProvider.provider == prov.value, - ).first() - if ai_prov: - default_models = { - ModelProvider.OPENAI: "gpt-5-mini", - ModelProvider.ANTHROPIC: "claude-sonnet-4-20250514", - ModelProvider.GOOGLE: "gemini-2.0-flash", - } - return prov, model or default_models.get(prov, "gpt-5-mini") - - raise HTTPException( - 400, - "No active AI provider configured. Add an OpenAI, Anthropic, or Google provider in AI Providers settings.", - ) +from app.services.ai.llm_resolver import get_llm_provider_and_model as _get_llm_provider_and_model @router.post("/generate-description") diff --git a/app/api/v1/routes/aiproviders.py b/app/api/v1/routes/aiproviders.py index 0d44f90..61deb31 100644 --- a/app/api/v1/routes/aiproviders.py +++ b/app/api/v1/routes/aiproviders.py @@ -17,48 +17,129 @@ from app.dependencies import get_db, get_organization_id from app.models.database import AIProvider +from app.models.enums import CredentialRoutingMode from app.models.schemas import ( AIProviderCreate, AIProviderUpdate, AIProviderResponse ) +from app.config import settings from app.core.encryption import encrypt_api_key from app.services.credentials.resolver import clear_other_defaults from app.services.ai.llm_gateway import ( GATEWAY_MANAGED_KEY_SENTINEL, - gateway_managed_credentials_enabled, + get_credential_effective_gateway_interface, + get_credential_effective_routing_label, is_gateway_managed_stored_key, + normalize_bifrost_native_url, + normalize_bifrost_url, ) router = APIRouter(prefix="/aiproviders", tags=["aiproviders"]) -def _scrub_for_response(db: Session, instance: AIProvider) -> AIProviderResponse: - """Detach the row from the session before clearing ``api_key``. +def _sanitize_gateway_base_url( + base_url: Optional[str], + gateway_interface: str, +) -> Optional[str]: + trimmed = (base_url or "").strip() + if not trimmed: + return None + interface = (gateway_interface or "inherit").strip().lower() + try: + if interface == "native_openai": + return normalize_bifrost_native_url(trimmed) or None + if interface == "litellm_shim": + return normalize_bifrost_url(trimmed) or None + return trimmed + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + - The same SQLAlchemy session is reused across requests in tests; if we - leave the instance attached and assign ``api_key = None``, SQLAlchemy - treats it as a pending UPDATE and tries to flush ``api_key=NULL`` on - the next request — which violates the column's NOT NULL constraint. - """ +def _scrub_for_response( + db: Session, + instance: AIProvider, + organization_id: UUID, +) -> AIProviderResponse: + """Detach the row from the session before clearing ``api_key``.""" gateway_managed = is_gateway_managed_stored_key(instance.api_key) + effective_routing = get_credential_effective_routing_label( + organization_id, + db, + instance.routing_mode, + ) + effective_gateway_interface = get_credential_effective_gateway_interface( + organization_id, + db, + getattr(instance, "gateway_interface", None), + ) + has_gateway_auth_secret = bool(getattr(instance, "gateway_auth_secret", None)) db.expunge(instance) instance.api_key = None + instance.gateway_auth_secret = None response = AIProviderResponse.model_validate(instance) - return response.model_copy(update={"gateway_managed": gateway_managed}) + return response.model_copy( + update={ + "gateway_managed": gateway_managed, + "effective_routing": effective_routing, + "effective_gateway_interface": effective_gateway_interface, + "has_gateway_auth_secret": has_gateway_auth_secret, + } + ) -def _encrypt_provider_api_key(api_key: Optional[str]) -> str: +def _validate_routing_and_api_key( + *, + routing_mode: CredentialRoutingMode, + api_key: Optional[str], + gateway_model: Optional[str], + has_existing_key: bool = False, +) -> None: + mode = routing_mode.value if hasattr(routing_mode, "value") else str(routing_mode) + trimmed_key = (api_key or "").strip() + + if mode == CredentialRoutingMode.DIRECT.value and not trimmed_key and not has_existing_key: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="api_key is required when routing_mode is direct.", + ) + + if mode == CredentialRoutingMode.GATEWAY.value: + return + + if mode == CredentialRoutingMode.INHERIT.value: + if not trimmed_key and not has_existing_key: + if settings.LLM_GATEWAY_PASSTHROUGH_PROVIDER_KEYS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "api_key is required when platform passthrough_provider_keys " + "is enabled." + ), + ) + return + + +def _encrypt_provider_api_key( + api_key: Optional[str], + *, + routing_mode: CredentialRoutingMode, +) -> str: trimmed = (api_key or "").strip() if trimmed: return encrypt_api_key(trimmed) - if gateway_managed_credentials_enabled(): + + mode = routing_mode.value if hasattr(routing_mode, "value") else str(routing_mode) + if mode in ( + CredentialRoutingMode.GATEWAY.value, + CredentialRoutingMode.INHERIT.value, + ): return encrypt_api_key(GATEWAY_MANAGED_KEY_SENTINEL) + raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "api_key is required. When LLM gateway-managed credentials " - "are enabled (passthrough_provider_keys: false), you can omit " - "the key and provider secrets will be resolved by the gateway." - ), + detail="api_key is required when routing_mode is direct.", ) @@ -68,15 +149,15 @@ async def create_aiprovider( organization_id: UUID = Depends(get_organization_id), db: Session = Depends(get_db) ): - """Create a new AI Provider credential row. - - Multiple credentials per (org, provider) are now allowed. The first - row created for a given provider is auto-promoted to default; - subsequent rows are stored as additional keys and can be promoted via - ``set-default``. - """ + """Create a new AI Provider credential row.""" provider_value = aiprovider.provider.value if hasattr(aiprovider.provider, 'value') else aiprovider.provider + _validate_routing_and_api_key( + routing_mode=aiprovider.routing_mode, + api_key=aiprovider.api_key, + gateway_model=aiprovider.gateway_model, + ) + existing_default = db.query(AIProvider).filter( AIProvider.organization_id == organization_id, func.lower(AIProvider.provider) == provider_value.lower(), @@ -86,13 +167,32 @@ async def create_aiprovider( requested_default = bool(aiprovider.is_default) will_be_default = requested_default or existing_default is None - encrypted_api_key = _encrypt_provider_api_key(aiprovider.api_key) + encrypted_api_key = _encrypt_provider_api_key( + aiprovider.api_key, + routing_mode=aiprovider.routing_mode, + ) + gateway_interface_value = aiprovider.gateway_interface.value db_aiprovider = AIProvider( organization_id=organization_id, provider=provider_value, api_key=encrypted_api_key, name=aiprovider.name, is_default=will_be_default, + routing_mode=aiprovider.routing_mode.value, + gateway_model=aiprovider.gateway_model, + gateway_interface=gateway_interface_value, + gateway_base_url=_sanitize_gateway_base_url( + aiprovider.gateway_base_url, + gateway_interface_value, + ), + gateway_auth_header=aiprovider.gateway_auth_header, + gateway_auth_secret_env=aiprovider.gateway_auth_secret_env, + gateway_auth_secret=( + encrypt_api_key(aiprovider.gateway_auth_secret) + if aiprovider.gateway_auth_secret + else None + ), + gateway_extra_headers=aiprovider.gateway_extra_headers, ) db.add(db_aiprovider) db.flush() @@ -110,7 +210,7 @@ async def create_aiprovider( db.commit() db.refresh(db_aiprovider) - return _scrub_for_response(db, db_aiprovider) + return _scrub_for_response(db, db_aiprovider, organization_id) @router.get("", response_model=List[AIProviderResponse], operation_id="listAIProviders") @@ -126,7 +226,10 @@ async def list_aiproviders( .all() ) - return [_scrub_for_response(db, provider) for provider in aiproviders] + return [ + _scrub_for_response(db, provider, organization_id) + for provider in aiproviders + ] @router.get("/{aiprovider_id}", response_model=AIProviderResponse) @@ -146,7 +249,7 @@ async def get_aiprovider( status_code=404, detail=f"AI Provider {aiprovider_id} not found" ) - return _scrub_for_response(db, aiprovider) + return _scrub_for_response(db, aiprovider, organization_id) @router.put("/{aiprovider_id}", response_model=AIProviderResponse, operation_id="updateAIProvider") @@ -168,17 +271,59 @@ async def update_aiprovider( ) update_data = aiprovider_update.model_dump(exclude_unset=True) + next_routing_mode = update_data.get( + "routing_mode", + CredentialRoutingMode(db_aiprovider.routing_mode), + ) + next_gateway_model = update_data.get("gateway_model", db_aiprovider.gateway_model) + next_api_key = update_data.get("api_key") + + if "routing_mode" in update_data or "api_key" in update_data or "gateway_model" in update_data: + _validate_routing_and_api_key( + routing_mode=next_routing_mode, + api_key=next_api_key, + gateway_model=next_gateway_model, + has_existing_key=( + bool(db_aiprovider.api_key) + and not is_gateway_managed_stored_key(db_aiprovider.api_key) + ), + ) + + skip_fields = { + "api_key", + "routing_mode", + "gateway_interface", + "gateway_auth_secret", + "clear_gateway_auth_secret", + } + for field, value in update_data.items(): - if field == 'api_key' and value: - encrypted_api_key = encrypt_api_key(value) - db_aiprovider.api_key = encrypted_api_key - else: - setattr(db_aiprovider, field, value) + if field in skip_fields: + continue + if field == "gateway_base_url": + interface = ( + update_data["gateway_interface"].value + if update_data.get("gateway_interface") is not None + else db_aiprovider.gateway_interface + ) + value = _sanitize_gateway_base_url(value, interface) + setattr(db_aiprovider, field, value) + + if "api_key" in update_data and update_data["api_key"]: + db_aiprovider.api_key = encrypt_api_key(update_data["api_key"]) + if "routing_mode" in update_data and update_data["routing_mode"] is not None: + db_aiprovider.routing_mode = update_data["routing_mode"].value + if "gateway_interface" in update_data and update_data["gateway_interface"] is not None: + db_aiprovider.gateway_interface = update_data["gateway_interface"].value + if update_data.get("clear_gateway_auth_secret"): + db_aiprovider.gateway_auth_secret = None + elif update_data.get("gateway_auth_secret"): + db_aiprovider.gateway_auth_secret = encrypt_api_key(update_data["gateway_auth_secret"]) db.commit() db.refresh(db_aiprovider) - return _scrub_for_response(db, db_aiprovider) + return _scrub_for_response(db, db_aiprovider, organization_id) @router.post( @@ -219,7 +364,7 @@ async def set_default_aiprovider( db.commit() db.refresh(db_aiprovider) - return _scrub_for_response(db, db_aiprovider) + return _scrub_for_response(db, db_aiprovider, organization_id) @router.delete("/{aiprovider_id}", status_code=status.HTTP_204_NO_CONTENT, operation_id="deleteAIProvider") diff --git a/app/api/v1/routes/integrations.py b/app/api/v1/routes/integrations.py index 820287f..173f1a9 100644 --- a/app/api/v1/routes/integrations.py +++ b/app/api/v1/routes/integrations.py @@ -18,10 +18,25 @@ from app.core.encryption import encrypt_api_key, decrypt_api_key from app.services.credentials.resolver import clear_other_defaults from app.services.voice_providers import get_voice_provider +from app.services.ai.llm_gateway import get_credential_effective_routing_label router = APIRouter(prefix="/integrations", tags=["Integrations"]) +def _integration_response( + integration: Integration, + organization_id: UUID, + db: Session, +) -> IntegrationResponse: + effective_routing = get_credential_effective_routing_label( + organization_id, + db, + integration.routing_mode, + ) + response = IntegrationResponse.model_validate(integration) + return response.model_copy(update={"effective_routing": effective_routing}) + + def _validate_smallest_connection(raw_api_key: str): """Validate a Smallest key via GET /atoms/v1/user.""" try: @@ -87,6 +102,7 @@ async def create_integration( public_key=integration_data.public_key, is_active=True, is_default=will_be_default, + routing_mode=integration_data.routing_mode.value, last_tested_at=datetime.now(timezone.utc) if user_details is not None else None, ) @@ -106,7 +122,7 @@ async def create_integration( db.commit() db.refresh(integration) - return integration + return _integration_response(integration, organization_id, db) @router.post( @@ -156,7 +172,7 @@ async def set_default_integration( integration.is_default = True db.commit() db.refresh(integration) - return integration + return _integration_response(integration, organization_id, db) @router.get("", response_model=List[IntegrationResponse], operation_id="listIntegrations") @@ -190,7 +206,10 @@ async def list_integrations( integration.platform, ) - return filtered_integrations + return [ + _integration_response(integration, organization_id, db) + for integration in filtered_integrations + ] @router.get("/{integration_id}", response_model=IntegrationResponse) @@ -220,7 +239,7 @@ async def get_integration( if raw_platform not in {p.value for p in IntegrationPlatform}: raise HTTPException(status_code=404, detail="Integration not found") - return integration + return _integration_response(integration, organization_id, db) @router.put("/{integration_id}", response_model=IntegrationResponse, operation_id="updateIntegration") @@ -258,11 +277,14 @@ async def update_integration( if integration_update.is_active is not None: integration.is_active = integration_update.is_active + + if integration_update.routing_mode is not None: + integration.routing_mode = integration_update.routing_mode.value db.commit() db.refresh(integration) - return integration + return _integration_response(integration, organization_id, db) @router.delete("/{integration_id}", operation_id="deleteIntegration") diff --git a/app/api/v1/routes/llm_gateway.py b/app/api/v1/routes/llm_gateway.py index 65924b8..248ab6e 100644 --- a/app/api/v1/routes/llm_gateway.py +++ b/app/api/v1/routes/llm_gateway.py @@ -9,6 +9,7 @@ from app.dependencies import get_db, get_organization_id from app.services.ai.llm_gateway_settings import ( + GatewayInterfaceOverride, GatewayMode, GatewayTypeOverride, get_org_settings, @@ -21,14 +22,17 @@ class LLMGatewaySettingsResponse(BaseModel): mode: GatewayMode gateway_type: GatewayTypeOverride + gateway_interface: GatewayInterfaceOverride = "inherit" base_url: Optional[str] = None has_virtual_key: bool = False has_master_key: bool = False platform_enabled: bool = False platform_gateway_type: Literal["bifrost", "litellm_proxy"] = "bifrost" + platform_gateway_interface: Literal["litellm_shim", "native_openai"] = "litellm_shim" platform_base_url: Optional[str] = None effective_routing: Literal["direct", "bifrost", "litellm_proxy"] effective_gateway_type: Optional[Literal["bifrost", "litellm_proxy"]] = None + effective_gateway_interface: Optional[Literal["litellm_shim", "native_openai"]] = None effective_base_url: Optional[str] = None effective_has_virtual_key: bool = False effective_has_master_key: bool = False @@ -44,6 +48,10 @@ class LLMGatewaySettingsUpdate(BaseModel): default="inherit", description="inherit: use platform gateway type; bifrost or litellm_proxy to override", ) + gateway_interface: GatewayInterfaceOverride = Field( + default="inherit", + description="inherit: use platform default; litellm_shim or native_openai for Bifrost API surface", + ) base_url: Optional[str] = Field( default=None, description="Optional org-specific gateway URL override.", @@ -85,6 +93,7 @@ def update_llm_gateway_settings( db, mode=body.mode, gateway_type=body.gateway_type, + gateway_interface=body.gateway_interface, base_url=body.base_url, virtual_key=body.virtual_key, master_key=body.master_key, diff --git a/app/api/v1/routes/prompt_partials.py b/app/api/v1/routes/prompt_partials.py index f167259..b336540 100644 --- a/app/api/v1/routes/prompt_partials.py +++ b/app/api/v1/routes/prompt_partials.py @@ -40,6 +40,7 @@ class GeneratePromptRequest(BaseModel): format_style: Optional[str] = "structured" provider: Optional[str] = None model: Optional[str] = None + credential_id: Optional[UUID] = None llm_config: Optional[Dict[str, Any]] = None @@ -48,6 +49,7 @@ class ImprovePromptRequest(BaseModel): instructions: Optional[str] = None provider: Optional[str] = None model: Optional[str] = None + credential_id: Optional[UUID] = None llm_config: Optional[Dict[str, Any]] = None @@ -192,7 +194,7 @@ async def generate_prompt_with_ai( raise HTTPException(400, "Description is required") provider_enum, model_str = _get_llm_provider_and_model( - organization_id, db, data.provider, data.model + organization_id, db, data.provider, data.model, data.credential_id ) user_prompt = ( @@ -217,6 +219,7 @@ async def generate_prompt_with_ai( db=db, llm_config=data.llm_config, task_defaults={"temperature": 0.7, "max_tokens": 4000}, + credential_id=data.credential_id, ) return {"content": result["text"], "provider": provider_enum.value, "model": model_str} except Exception as e: @@ -238,7 +241,7 @@ async def improve_prompt_with_ai( raise HTTPException(400, "Content is required") provider_enum, model_str = _get_llm_provider_and_model( - organization_id, db, data.provider, data.model + organization_id, db, data.provider, data.model, data.credential_id ) user_content = data.content @@ -259,6 +262,7 @@ async def improve_prompt_with_ai( db=db, llm_config=data.llm_config, task_defaults={"temperature": 0.3, "max_tokens": 4000}, + credential_id=data.credential_id, ) return {"content": result["text"], "provider": provider_enum.value, "model": model_str} except Exception as e: diff --git a/app/config.py b/app/config.py index 4bf5d00..8de0d12 100644 --- a/app/config.py +++ b/app/config.py @@ -175,6 +175,7 @@ class Settings(BaseSettings): LLM_GATEWAY_VIRTUAL_KEY: Optional[str] = None LLM_GATEWAY_MASTER_KEY: Optional[str] = None LLM_GATEWAY_PASSTHROUGH_PROVIDER_KEYS: bool = True + LLM_GATEWAY_INTERFACE: str = "litellm_shim" # litellm_shim | native_openai model_config = SettingsConfigDict( env_file=".env", @@ -631,6 +632,10 @@ def _apply_llm_gateway_settings(gateway_cfg: dict, *, gateway_type: str) -> None settings.LLM_GATEWAY_PASSTHROUGH_PROVIDER_KEYS = bool( gateway_cfg["passthrough_provider_keys"] ) + if gateway_cfg.get("gateway_interface"): + interface = str(gateway_cfg["gateway_interface"]).strip().lower() + if interface in ("litellm_shim", "native_openai"): + settings.LLM_GATEWAY_INTERFACE = interface if "llm_gateway" in config_data: llm_cfg = config_data["llm_gateway"] diff --git a/app/migrations/053_integration_routing_settings.py b/app/migrations/053_integration_routing_settings.py new file mode 100644 index 0000000..be6800e --- /dev/null +++ b/app/migrations/053_integration_routing_settings.py @@ -0,0 +1,77 @@ +""" +Migration: Per-integration LLM gateway routing settings. + +Adds ``routing_mode`` and ``gateway_model`` to aiproviders, and +``routing_mode`` to integrations (voice platforms). +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Add per-credential routing_mode and gateway_model for AI providers " + "and voice platform integrations." +) + + +def _column_exists(db: Session, table_name: str, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table_name, "column_name": column_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _column_exists(db, "aiproviders", "routing_mode"): + db.execute( + text( + """ + ALTER TABLE aiproviders + ADD COLUMN routing_mode VARCHAR(20) NOT NULL DEFAULT 'inherit' + """ + ) + ) + print("Added aiproviders.routing_mode (varchar, default 'inherit')") + + if not _column_exists(db, "aiproviders", "gateway_model"): + db.execute( + text( + """ + ALTER TABLE aiproviders + ADD COLUMN gateway_model VARCHAR(255) NULL + """ + ) + ) + print("Added aiproviders.gateway_model (varchar, nullable)") + + if not _column_exists(db, "integrations", "routing_mode"): + db.execute( + text( + """ + ALTER TABLE integrations + ADD COLUMN routing_mode VARCHAR(20) NOT NULL DEFAULT 'inherit' + """ + ) + ) + print("Added integrations.routing_mode (varchar, default 'inherit')") + + +def downgrade(db: Session): + if _column_exists(db, "integrations", "routing_mode"): + db.execute(text("ALTER TABLE integrations DROP COLUMN routing_mode")) + print("Dropped integrations.routing_mode") + + if _column_exists(db, "aiproviders", "gateway_model"): + db.execute(text("ALTER TABLE aiproviders DROP COLUMN gateway_model")) + print("Dropped aiproviders.gateway_model") + + if _column_exists(db, "aiproviders", "routing_mode"): + db.execute(text("ALTER TABLE aiproviders DROP COLUMN routing_mode")) + print("Dropped aiproviders.routing_mode") diff --git a/app/migrations/054_gateway_interface_settings.py b/app/migrations/054_gateway_interface_settings.py new file mode 100644 index 0000000..73a5bcb --- /dev/null +++ b/app/migrations/054_gateway_interface_settings.py @@ -0,0 +1,62 @@ +""" +Migration: Bifrost gateway interface and per-credential base URL overrides. + +Adds ``gateway_interface`` and ``gateway_base_url`` to aiproviders for +native OpenAI-compatible Bifrost routing vs the /litellm shim. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Add gateway_interface and gateway_base_url on aiproviders for " + "Bifrost native vs LiteLLM shim routing." +) + + +def _column_exists(db: Session, table_name: str, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table_name, "column_name": column_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _column_exists(db, "aiproviders", "gateway_interface"): + db.execute( + text( + """ + ALTER TABLE aiproviders + ADD COLUMN gateway_interface VARCHAR(20) NOT NULL DEFAULT 'inherit' + """ + ) + ) + print("Added aiproviders.gateway_interface (varchar, default 'inherit')") + + if not _column_exists(db, "aiproviders", "gateway_base_url"): + db.execute( + text( + """ + ALTER TABLE aiproviders + ADD COLUMN gateway_base_url VARCHAR(512) NULL + """ + ) + ) + print("Added aiproviders.gateway_base_url (varchar, nullable)") + + +def downgrade(db: Session): + if _column_exists(db, "aiproviders", "gateway_base_url"): + db.execute(text("ALTER TABLE aiproviders DROP COLUMN gateway_base_url")) + print("Dropped aiproviders.gateway_base_url") + + if _column_exists(db, "aiproviders", "gateway_interface"): + db.execute(text("ALTER TABLE aiproviders DROP COLUMN gateway_interface")) + print("Dropped aiproviders.gateway_interface") diff --git a/app/migrations/055_gateway_auth_overrides.py b/app/migrations/055_gateway_auth_overrides.py new file mode 100644 index 0000000..7bf8c74 --- /dev/null +++ b/app/migrations/055_gateway_auth_overrides.py @@ -0,0 +1,77 @@ +""" +Migration: Per-credential Bifrost gateway auth header and secret overrides. + +Allows each AI Provider to specify a custom auth header name, an environment +variable name for the secret, or an encrypted inline gateway auth secret. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Add gateway_auth_header, gateway_auth_secret_env, and gateway_auth_secret " + "on aiproviders for per-credential Bifrost authentication." +) + + +def _column_exists(db: Session, table_name: str, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table_name, "column_name": column_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _column_exists(db, "aiproviders", "gateway_auth_header"): + db.execute( + text( + """ + ALTER TABLE aiproviders + ADD COLUMN gateway_auth_header VARCHAR(64) NULL + """ + ) + ) + print("Added aiproviders.gateway_auth_header (varchar, nullable)") + + if not _column_exists(db, "aiproviders", "gateway_auth_secret_env"): + db.execute( + text( + """ + ALTER TABLE aiproviders + ADD COLUMN gateway_auth_secret_env VARCHAR(128) NULL + """ + ) + ) + print("Added aiproviders.gateway_auth_secret_env (varchar, nullable)") + + if not _column_exists(db, "aiproviders", "gateway_auth_secret"): + db.execute( + text( + """ + ALTER TABLE aiproviders + ADD COLUMN gateway_auth_secret VARCHAR NULL + """ + ) + ) + print("Added aiproviders.gateway_auth_secret (varchar, nullable, encrypted at rest)") + + +def downgrade(db: Session): + if _column_exists(db, "aiproviders", "gateway_auth_secret"): + db.execute(text("ALTER TABLE aiproviders DROP COLUMN gateway_auth_secret")) + print("Dropped aiproviders.gateway_auth_secret") + + if _column_exists(db, "aiproviders", "gateway_auth_secret_env"): + db.execute(text("ALTER TABLE aiproviders DROP COLUMN gateway_auth_secret_env")) + print("Dropped aiproviders.gateway_auth_secret_env") + + if _column_exists(db, "aiproviders", "gateway_auth_header"): + db.execute(text("ALTER TABLE aiproviders DROP COLUMN gateway_auth_header")) + print("Dropped aiproviders.gateway_auth_header") diff --git a/app/migrations/056_gateway_extra_headers.py b/app/migrations/056_gateway_extra_headers.py new file mode 100644 index 0000000..a17fbed --- /dev/null +++ b/app/migrations/056_gateway_extra_headers.py @@ -0,0 +1,44 @@ +""" +Migration: Arbitrary per-credential gateway HTTP headers. + +Adds ``gateway_extra_headers`` JSON on aiproviders for custom headers sent +with every gateway-routed LiteLLM call. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add gateway_extra_headers JSON on aiproviders for custom gateway headers." + + +def _column_exists(db: Session, table_name: str, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table_name, "column_name": column_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _column_exists(db, "aiproviders", "gateway_extra_headers"): + db.execute( + text( + """ + ALTER TABLE aiproviders + ADD COLUMN gateway_extra_headers JSON NULL + """ + ) + ) + print("Added aiproviders.gateway_extra_headers (json, nullable)") + + +def downgrade(db: Session): + if _column_exists(db, "aiproviders", "gateway_extra_headers"): + db.execute(text("ALTER TABLE aiproviders DROP COLUMN gateway_extra_headers")) + print("Dropped aiproviders.gateway_extra_headers") diff --git a/app/models/database.py b/app/models/database.py index 50e3217..d60e4d2 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -556,6 +556,8 @@ class Integration(Base): # A partial unique index in migration 028 enforces at most one default # per (org, platform) at the DB level. is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated @@ -627,6 +629,22 @@ class AIProvider(Base): # marks the row resolved when no explicit credential id is selected. # A partial unique index in migration 028 enforces at most one default. is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Bifrost custom model ID used when routing via gateway + gateway_model = Column(String(255), nullable=True) + # inherit | litellm_shim | native_openai — Bifrost API surface override + gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Optional per-credential Bifrost/gateway base URL override + gateway_base_url = Column(String(512), nullable=True) + # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) + gateway_auth_header = Column(String(64), nullable=True) + # Env var name whose value is sent as the gateway auth secret + gateway_auth_secret_env = Column(String(128), nullable=True) + # Encrypted inline gateway auth secret (alternative to env var) + gateway_auth_secret = Column(String, nullable=True) + # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls + gateway_extra_headers = Column(JSON, nullable=True) created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated diff --git a/app/models/enums.py b/app/models/enums.py index dba91cf..e0df56c 100644 --- a/app/models/enums.py +++ b/app/models/enums.py @@ -108,6 +108,21 @@ class TelephonyProvider(str, enum.Enum): PLIVO = "plivo" EXOTEL = "exotel" + +class CredentialRoutingMode(str, enum.Enum): + """Per-credential LLM routing preference.""" + INHERIT = "inherit" + GATEWAY = "gateway" + DIRECT = "direct" + + +class GatewayInterfaceMode(str, enum.Enum): + """How Bifrost is reached when gateway routing is active.""" + INHERIT = "inherit" + LITELLM_SHIM = "litellm_shim" + NATIVE_OPENAI = "native_openai" + + class ModelProvider(str, enum.Enum): """Model provider enumeration for extensibility.""" OPENAI = "openai" diff --git a/app/models/schemas.py b/app/models/schemas.py index ae2513e..d67d9e5 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -1,13 +1,14 @@ """Pydantic schemas for request/response validation.""" from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, validator +import re from typing import Optional, List, Dict, Any, Literal from datetime import date, datetime from uuid import UUID from app.models.enums import ( EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, - IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, + IntegrationPlatform, ModelProvider, CredentialRoutingMode, GatewayInterfaceMode, VoiceBundleType, TestAgentConversationStatus, MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, CallImportStatus, CallImportRowStatus, CallImportParameterType, @@ -555,6 +556,10 @@ class IntegrationCreate(BaseModel): api_key: str = Field(..., description="Private API key for the platform") public_key: Optional[str] = Field(None, description="Optional public API key (e.g. for Vapi)") name: Optional[str] = Field(None, description="Optional friendly name for the integration") + routing_mode: CredentialRoutingMode = Field( + CredentialRoutingMode.INHERIT, + description="LLM routing preference: inherit org default, force gateway, or direct API key.", + ) is_default: Optional[bool] = Field( None, description=( @@ -570,6 +575,7 @@ class IntegrationUpdate(BaseModel): api_key: Optional[str] = None public_key: Optional[str] = None is_active: Optional[bool] = None + routing_mode: Optional[CredentialRoutingMode] = None class IntegrationResponse(BaseModel): @@ -581,6 +587,8 @@ class IntegrationResponse(BaseModel): public_key: Optional[str] = None is_active: bool is_default: bool = False + routing_mode: CredentialRoutingMode = CredentialRoutingMode.INHERIT + effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" created_at: datetime updated_at: datetime last_tested_at: Optional[datetime] = None @@ -605,6 +613,18 @@ def convert_platform(cls, v): raise ValueError(f"Invalid IntegrationPlatform value: {v}") return v + @field_validator('routing_mode', mode='before') + @classmethod + def convert_routing_mode(cls, v): + if v is None: + return CredentialRoutingMode.INHERIT + if isinstance(v, str): + try: + return CredentialRoutingMode(v.lower()) + except ValueError: + return CredentialRoutingMode.INHERIT + return v + model_config = ConfigDict(from_attributes=True) @@ -666,17 +686,86 @@ class S3UploadResponse(BaseModel): # AIProvider Schemas +_MAX_GATEWAY_EXTRA_HEADERS = 20 + + +def _validate_gateway_extra_headers( + value: Optional[Dict[str, Any]], +) -> Optional[Dict[str, str]]: + if value is None: + return None + if not isinstance(value, dict): + raise ValueError("gateway_extra_headers must be a JSON object of string keys and values.") + if len(value) > _MAX_GATEWAY_EXTRA_HEADERS: + raise ValueError( + f"gateway_extra_headers supports at most {_MAX_GATEWAY_EXTRA_HEADERS} headers." + ) + normalized: Dict[str, str] = {} + for raw_key, raw_val in value.items(): + key = str(raw_key).strip() + if not key: + raise ValueError("gateway_extra_headers keys must be non-empty strings.") + if len(key) > 64 or any(ch.isspace() for ch in key): + raise ValueError(f"Invalid gateway header name: {key!r}") + if raw_val is None: + raise ValueError(f"gateway_extra_headers[{key!r}] must be a string value.") + val = str(raw_val).strip() + if not val: + raise ValueError(f"gateway_extra_headers[{key!r}] must be a non-empty string.") + if len(val) > 1024 or "\n" in val or "\r" in val: + raise ValueError(f"gateway_extra_headers[{key!r}] value is invalid.") + normalized[key] = val + return normalized or None + + class AIProviderCreate(BaseModel): """Schema for creating an AI Provider.""" provider: ModelProvider api_key: Optional[str] = Field( None, description=( - "Provider API key. Optional when the platform uses Bifrost " + "Provider API key. Optional when routing via gateway with " "gateway-managed credentials (passthrough_provider_keys: false)." ), ) name: Optional[str] = None + routing_mode: CredentialRoutingMode = Field( + CredentialRoutingMode.INHERIT, + description="LLM routing preference: inherit org default, force gateway, or direct API key.", + ) + gateway_model: Optional[str] = Field( + None, + min_length=1, + max_length=255, + description="Bifrost custom model ID sent when routing via gateway.", + ) + gateway_interface: GatewayInterfaceMode = Field( + GatewayInterfaceMode.INHERIT, + description="Bifrost API surface: inherit org default, LiteLLM shim, or native OpenAI-compatible.", + ) + gateway_base_url: Optional[str] = Field( + None, + max_length=512, + description="Optional per-credential Bifrost/gateway base URL override.", + ) + gateway_auth_header: Optional[str] = Field( + None, + max_length=64, + description="Auth header name for Bifrost (default x-bf-vk).", + ) + gateway_auth_secret_env: Optional[str] = Field( + None, + max_length=128, + description="Environment variable name holding the gateway auth secret.", + ) + gateway_auth_secret: Optional[str] = Field( + None, + description="Inline gateway auth secret (encrypted at rest). Alternative to env var.", + ) + gateway_extra_headers: Optional[Dict[str, str]] = Field( + None, + description="Arbitrary HTTP headers sent with gateway-routed LiteLLM calls.", + ) is_default: Optional[bool] = Field( None, description=( @@ -693,12 +782,133 @@ def validate_api_key(cls, v: Optional[str]) -> Optional[str]: trimmed = v.strip() return trimmed or None + @field_validator("gateway_model") + @classmethod + def validate_gateway_model(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_base_url") + @classmethod + def validate_gateway_base_url(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_auth_header") + @classmethod + def validate_gateway_auth_header(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if len(trimmed) > 64 or any(ch.isspace() for ch in trimmed): + raise ValueError("gateway_auth_header must be a single non-empty header name.") + return trimmed + + @field_validator("gateway_auth_secret_env") + @classmethod + def validate_gateway_auth_secret_env(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", trimmed): + raise ValueError( + "gateway_auth_secret_env must be a valid environment variable name." + ) + return trimmed + + @field_validator("gateway_auth_secret") + @classmethod + def validate_gateway_auth_secret(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_extra_headers") + @classmethod + def validate_gateway_extra_headers(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, str]]: + return _validate_gateway_extra_headers(v) + class AIProviderUpdate(BaseModel): """Schema for updating an AI Provider.""" api_key: Optional[str] = Field(None, min_length=1) name: Optional[str] = None is_active: Optional[bool] = None + routing_mode: Optional[CredentialRoutingMode] = None + gateway_model: Optional[str] = Field(None, min_length=1, max_length=255) + gateway_interface: Optional[GatewayInterfaceMode] = None + gateway_base_url: Optional[str] = Field(None, max_length=512) + gateway_auth_header: Optional[str] = Field(None, max_length=64) + gateway_auth_secret_env: Optional[str] = Field(None, max_length=128) + gateway_auth_secret: Optional[str] = None + clear_gateway_auth_secret: bool = False + gateway_extra_headers: Optional[Dict[str, str]] = None + + @field_validator("gateway_model") + @classmethod + def validate_gateway_model(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_base_url") + @classmethod + def validate_gateway_base_url_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_auth_header") + @classmethod + def validate_gateway_auth_header_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if len(trimmed) > 64 or any(ch.isspace() for ch in trimmed): + raise ValueError("gateway_auth_header must be a single non-empty header name.") + return trimmed + + @field_validator("gateway_auth_secret_env") + @classmethod + def validate_gateway_auth_secret_env_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", trimmed): + raise ValueError( + "gateway_auth_secret_env must be a valid environment variable name." + ) + return trimmed + + @field_validator("gateway_auth_secret") + @classmethod + def validate_gateway_auth_secret_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_extra_headers") + @classmethod + def validate_gateway_extra_headers_update( + cls, v: Optional[Dict[str, Any]] + ) -> Optional[Dict[str, str]]: + return _validate_gateway_extra_headers(v) class AIProviderResponse(BaseModel): @@ -709,7 +919,17 @@ class AIProviderResponse(BaseModel): name: Optional[str] is_active: bool is_default: bool = False + routing_mode: CredentialRoutingMode = CredentialRoutingMode.INHERIT + gateway_model: Optional[str] = None + gateway_interface: GatewayInterfaceMode = GatewayInterfaceMode.INHERIT + gateway_base_url: Optional[str] = None + gateway_auth_header: Optional[str] = None + gateway_auth_secret_env: Optional[str] = None + has_gateway_auth_secret: bool = False + gateway_extra_headers: Optional[Dict[str, str]] = None gateway_managed: bool = False + effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" + effective_gateway_interface: Literal["litellm_shim", "native_openai"] = "litellm_shim" created_at: datetime updated_at: datetime last_tested_at: Optional[datetime] @@ -730,6 +950,18 @@ def convert_provider(cls, v): return enum_member raise ValueError(f"Invalid ModelProvider value: {v}") return v + + @field_validator('routing_mode', mode='before') + @classmethod + def convert_routing_mode(cls, v): + if v is None: + return CredentialRoutingMode.INHERIT + if isinstance(v, str): + try: + return CredentialRoutingMode(v.lower()) + except ValueError: + return CredentialRoutingMode.INHERIT + return v model_config = ConfigDict(from_attributes=True) diff --git a/app/services/ai/llm_gateway.py b/app/services/ai/llm_gateway.py index 85d41b1..3421f26 100644 --- a/app/services/ai/llm_gateway.py +++ b/app/services/ai/llm_gateway.py @@ -3,13 +3,15 @@ Supports Bifrost (/litellm proxy) and self-hosted LiteLLM Proxy gateways. Platform defaults live in ``config.yml``; per-org overrides are stored in -``organizations.llm_gateway_settings`` JSON. +``organizations.llm_gateway_settings`` JSON. Per-credential overrides live on +``aiproviders`` and ``integrations`` rows. """ from __future__ import annotations +import os from dataclasses import dataclass -from typing import Any, Dict, Literal, Optional +from typing import Any, Dict, Literal, Optional, Tuple from urllib.parse import urlparse from uuid import UUID @@ -20,6 +22,9 @@ GatewayType = Literal["bifrost", "litellm_proxy"] +GatewayInterface = Literal["litellm_shim", "native_openai"] +GatewayInterfaceOverride = Literal["inherit", "litellm_shim", "native_openai"] +RoutingMode = Literal["inherit", "gateway", "direct"] EffectiveRouting = Literal["direct", "bifrost", "litellm_proxy"] # Backward-compatible alias used by stored AI provider credentials. @@ -44,14 +49,125 @@ def is_gateway_managed_stored_key(encrypted_api_key: str) -> bool: return False +@dataclass(frozen=True) +class CredentialRoutingContext: + """Per-credential routing preferences.""" + + routing_mode: RoutingMode = "inherit" + gateway_model: Optional[str] = None + gateway_interface: GatewayInterfaceOverride = "inherit" + gateway_base_url: Optional[str] = None + gateway_auth_header: Optional[str] = None + gateway_auth_secret_env: Optional[str] = None + gateway_auth_secret: Optional[str] = None + gateway_extra_headers: Optional[Dict[str, str]] = None + + +def _normalize_routing_mode(value: Any) -> RoutingMode: + mode = value.value if hasattr(value, "value") else value + mode = str(mode or "inherit").strip().lower() + if mode in ("inherit", "gateway", "direct"): + return mode # type: ignore[return-value] + return "inherit" + + +def _normalize_gateway_interface(value: Any) -> GatewayInterfaceOverride: + iface = value.value if hasattr(value, "value") else value + iface = str(iface or "inherit").strip().lower() + if iface in ("inherit", "litellm_shim", "native_openai"): + return iface # type: ignore[return-value] + return "inherit" + + +def _normalize_gateway_extra_headers(value: Any) -> Optional[Dict[str, str]]: + if not value: + return None + if not isinstance(value, dict): + return None + normalized: Dict[str, str] = {} + for raw_key, raw_val in value.items(): + key = str(raw_key).strip() + if not key: + continue + normalized[key] = str(raw_val) + return normalized or None + + +def _decrypt_gateway_auth_secret(encrypted: Any) -> Optional[str]: + if not encrypted: + return None + try: + from app.core.encryption import decrypt_api_key + + return decrypt_api_key(encrypted) + except Exception as exc: + logger.warning("Failed to decrypt credential gateway auth secret: {}", exc) + return None + + +def routing_context_from_ai_provider(provider: Any) -> CredentialRoutingContext: + """Build routing context from an ``AIProvider`` row.""" + gateway_model = getattr(provider, "gateway_model", None) + if gateway_model: + gateway_model = str(gateway_model).strip() or None + gateway_base_url = getattr(provider, "gateway_base_url", None) + if gateway_base_url: + gateway_base_url = str(gateway_base_url).strip() or None + gateway_auth_header = getattr(provider, "gateway_auth_header", None) + if gateway_auth_header: + gateway_auth_header = str(gateway_auth_header).strip() or None + gateway_auth_secret_env = getattr(provider, "gateway_auth_secret_env", None) + if gateway_auth_secret_env: + gateway_auth_secret_env = str(gateway_auth_secret_env).strip() or None + return CredentialRoutingContext( + routing_mode=_normalize_routing_mode(getattr(provider, "routing_mode", "inherit")), + gateway_model=gateway_model, + gateway_interface=_normalize_gateway_interface( + getattr(provider, "gateway_interface", "inherit") + ), + gateway_base_url=gateway_base_url, + gateway_auth_header=gateway_auth_header, + gateway_auth_secret_env=gateway_auth_secret_env, + gateway_auth_secret=_decrypt_gateway_auth_secret( + getattr(provider, "gateway_auth_secret", None) + ), + gateway_extra_headers=_normalize_gateway_extra_headers( + getattr(provider, "gateway_extra_headers", None) + ), + ) + + +def routing_context_from_integration(integration: Any) -> CredentialRoutingContext: + """Build routing context from a voice-platform ``Integration`` row.""" + return CredentialRoutingContext( + routing_mode=_normalize_routing_mode(getattr(integration, "routing_mode", "inherit")), + ) + + +def resolve_litellm_model( + *, + workload_model_str: str, + gateway_active: bool, + credential: Optional[CredentialRoutingContext], +) -> str: + """Pick a Bifrost custom model or use the workload-built model string.""" + if gateway_active and credential and credential.gateway_model: + return credential.gateway_model + return workload_model_str + + def resolve_litellm_api_key( organization_id: UUID, db: Session, ai_provider: Any, + *, + credential: Optional[CredentialRoutingContext] = None, ) -> Optional[str]: """Decrypt the org credential, or return None when the gateway supplies the key.""" from app.core.encryption import decrypt_api_key + ctx = credential or routing_context_from_ai_provider(ai_provider) + try: raw_key = decrypt_api_key(ai_provider.api_key) except Exception as exc: @@ -59,18 +175,21 @@ def resolve_litellm_api_key( f"Failed to decrypt API key for provider {ai_provider.provider}: {exc}" ) from exc - if raw_key != GATEWAY_MANAGED_KEY_SENTINEL: + gateway_config, effective = resolve_effective_routing(organization_id, db, ctx) + + if effective == "direct": + if raw_key == GATEWAY_MANAGED_KEY_SENTINEL: + raise RuntimeError( + "AI provider is configured for gateway-managed credentials, " + "but routing is set to direct. Add a provider API key or " + "change routing mode to gateway." + ) return raw_key - gateway = resolve_llm_gateway(organization_id, db) - if gateway and not gateway.passthrough_provider_keys: + if raw_key == GATEWAY_MANAGED_KEY_SENTINEL: return None - raise RuntimeError( - "AI provider is configured for gateway-managed credentials, " - "but the LLM gateway is not active for this organization. " - "Enable the LLM Gateway or add a provider API key." - ) + return raw_key @dataclass(frozen=True) @@ -78,8 +197,11 @@ class LLMGatewayConfig: """Resolved gateway settings for a single LiteLLM call.""" gateway_type: GatewayType + gateway_interface: GatewayInterface api_base: str virtual_key: Optional[str] = None + auth_header: str = "x-bf-vk" + extra_headers: Optional[Dict[str, str]] = None master_key: Optional[str] = None passthrough_provider_keys: bool = True @@ -104,6 +226,55 @@ def normalize_bifrost_url(base_url: str) -> str: return url +_NATIVE_OPENAI_PATH_SUFFIXES = ( + "/v1/chat/completions", + "/v1/completions", + "/chat/completions", + "/litellm", +) + + +def normalize_bifrost_native_url(base_url: str) -> str: + """Ensure a Bifrost native OpenAI-compatible URL is well-formed. + + LiteLLM passes ``api_base`` to the OpenAI client, which appends + ``/chat/completions`` (not ``/v1/chat/completions``). The base must + therefore end with ``/v1``, e.g. ``http://localhost:8080/v1``. + """ + url = (base_url or "").strip().rstrip("/") + if not url: + return "" + parsed = urlparse(url) + if not parsed.scheme or not parsed.netloc: + raise ValueError( + "Bifrost base_url must be an absolute URL including scheme and host " + "(e.g. http://localhost:8080)." + ) + original = url + stripped_paths = False + while True: + stripped = False + for suffix in _NATIVE_OPENAI_PATH_SUFFIXES: + if url.endswith(suffix): + url = url[: -len(suffix)] + stripped = True + stripped_paths = True + break + if not stripped: + break + if not url.endswith("/v1"): + url = f"{url}/v1" + if stripped_paths: + logger.warning( + "Normalized native Bifrost base_url '{}' -> '{}'. " + "Use the host root only (e.g. http://localhost:8080); " + "/v1 is appended automatically for LiteLLM.", + original, + url, + ) + return url + + def normalize_litellm_proxy_url(base_url: str) -> str: """Ensure a LiteLLM Proxy URL is well-formed.""" url = (base_url or "").strip().rstrip("/") @@ -118,8 +289,14 @@ def normalize_litellm_proxy_url(base_url: str) -> str: return url -def _normalize_base_url(base_url: str, gateway_type: GatewayType) -> str: +def _normalize_base_url( + base_url: str, + gateway_type: GatewayType, + gateway_interface: GatewayInterface, +) -> str: if gateway_type == "bifrost": + if gateway_interface == "native_openai": + return normalize_bifrost_native_url(base_url) return normalize_bifrost_url(base_url) return normalize_litellm_proxy_url(base_url) @@ -135,6 +312,7 @@ def _platform_config() -> Dict[str, Any]: "virtual_key": (settings.LLM_GATEWAY_VIRTUAL_KEY or "").strip() or None, "master_key": (settings.LLM_GATEWAY_MASTER_KEY or "").strip() or None, "passthrough_provider_keys": bool(settings.LLM_GATEWAY_PASSTHROUGH_PROVIDER_KEYS), + "gateway_interface": (settings.LLM_GATEWAY_INTERFACE or "litellm_shim").strip().lower(), } @@ -179,77 +357,224 @@ def _resolve_gateway_type(org: Dict[str, Any], platform: Dict[str, Any]) -> Gate return "bifrost" -def resolve_llm_gateway( - organization_id: UUID, - db: Session, -) -> Optional[LLMGatewayConfig]: - """Return effective LLM gateway config, or ``None`` for direct routing.""" - platform = _platform_config() - org = _get_org_raw_settings(organization_id, db) +def _resolve_gateway_interface( + credential: Optional[CredentialRoutingContext], + org: Dict[str, Any], + platform: Dict[str, Any], +) -> GatewayInterface: + if credential and credential.gateway_interface in ("litellm_shim", "native_openai"): + return credential.gateway_interface + + org_interface = org.get("gateway_interface") + if org_interface in ("litellm_shim", "native_openai"): + return org_interface + + platform_interface = platform.get("gateway_interface", "litellm_shim") + if platform_interface in ("litellm_shim", "native_openai"): + return platform_interface + + return "litellm_shim" + + +def _resolve_gateway_base_url( + credential: Optional[CredentialRoutingContext], + org: Dict[str, Any], + platform: Dict[str, Any], +) -> str: + if credential and credential.gateway_base_url: + return credential.gateway_base_url.strip() + return (org.get("base_url") or platform.get("base_url") or "").strip() + + +def _resolve_gateway_auth_header(credential: Optional[CredentialRoutingContext]) -> str: + if credential and credential.gateway_auth_header: + header = credential.gateway_auth_header.strip() + if header: + return header + return "x-bf-vk" + + +def _resolve_gateway_auth_secret( + credential: Optional[CredentialRoutingContext], + org: Dict[str, Any], + platform: Dict[str, Any], +) -> Optional[str]: + if credential and credential.gateway_auth_secret_env: + env_name = credential.gateway_auth_secret_env.strip() + if env_name: + secret = (os.environ.get(env_name) or "").strip() + if secret: + return secret + logger.warning( + "Gateway auth env var '{}' is unset or empty; falling back to org/platform keys.", + env_name, + ) + + if credential and credential.gateway_auth_secret: + return credential.gateway_auth_secret.strip() or None + + return _decrypt_org_virtual_key(org) or platform.get("virtual_key") + + +def _format_gateway_auth_header_value(header_name: str, secret: str) -> str: + if header_name.lower() == "authorization": + if secret.lower().startswith("bearer "): + return secret + return f"Bearer {secret}" + return secret + + +def _org_wants_gateway( + org: Dict[str, Any], + platform: Dict[str, Any], + *, + credential_mode: RoutingMode, +) -> bool: + """Return whether org/platform settings want gateway routing for inherit mode.""" + if credential_mode == "gateway": + return True + if credential_mode == "direct": + return False org_enabled = org.get("enabled") if org_enabled is False: - return None - + return False if org_enabled is True: - use_gateway = True - elif platform["enabled"]: - use_gateway = True - else: - return None + return True + return bool(platform["enabled"]) - if not use_gateway: - return None +def _build_gateway_config( + organization_id: UUID, + org: Dict[str, Any], + platform: Dict[str, Any], + *, + credential: Optional[CredentialRoutingContext], + credential_mode: RoutingMode, + strict: bool, +) -> Optional[LLMGatewayConfig]: + """Resolve gateway connection details from org/platform/credential settings.""" gateway_type = _resolve_gateway_type(org, platform) + gateway_interface = _resolve_gateway_interface(credential, org, platform) + base_url = _resolve_gateway_base_url(credential, org, platform) - base_url = (org.get("base_url") or platform["base_url"] or "").strip() if not base_url: - logger.warning( - "LLM gateway ({}) enabled for org {} but no base_url is configured; " - "falling back to direct provider routing.", - gateway_type, - organization_id, + message = ( + f"LLM gateway ({gateway_type}) is required for credential routing " + f"mode '{credential_mode}' but no base_url is configured for org " + f"{organization_id}." ) + if strict: + raise RuntimeError(message) + logger.warning("{} Falling back to direct provider routing.", message) return None try: - api_base = _normalize_base_url(base_url, gateway_type) + api_base = _normalize_base_url(base_url, gateway_type, gateway_interface) except ValueError as exc: - logger.warning( - "Invalid LLM gateway base_url for org {} ({}): {}; falling back to direct routing.", - organization_id, - gateway_type, - exc, + message = ( + f"Invalid LLM gateway base_url for org {organization_id} " + f"({gateway_type}): {exc}" ) + if strict: + raise RuntimeError(message) from exc + logger.warning("{} Falling back to direct provider routing.", message) return None - virtual_key = _decrypt_org_virtual_key(org) or platform["virtual_key"] + virtual_key = _resolve_gateway_auth_secret(credential, org, platform) + auth_header = _resolve_gateway_auth_header(credential) + extra_headers = ( + dict(credential.gateway_extra_headers) + if credential and credential.gateway_extra_headers + else None + ) master_key = _decrypt_org_master_key(org) or platform["master_key"] return LLMGatewayConfig( gateway_type=gateway_type, + gateway_interface=gateway_interface if gateway_type == "bifrost" else "litellm_shim", api_base=api_base, virtual_key=virtual_key, + auth_header=auth_header, + extra_headers=extra_headers, master_key=master_key, passthrough_provider_keys=platform["passthrough_provider_keys"], ) -# Providers whose LiteLLM handlers build native API paths (e.g. Gemini -# ``:generateContent``) when ``api_base`` is set. Gateways like Bifrost and -# LiteLLM Proxy expect OpenAI-compatible ``/v1/chat/completions`` instead. -_NATIVE_GATEWAY_MODEL_PREFIXES = ( - "gemini/", - "google/", - "vertex/", - "vertex_ai/", -) +def resolve_effective_routing( + organization_id: UUID, + db: Session, + credential: Optional[CredentialRoutingContext] = None, +) -> Tuple[Optional[LLMGatewayConfig], EffectiveRouting]: + """Merge org/platform gateway config with per-credential override.""" + platform = _platform_config() + org = _get_org_raw_settings(organization_id, db) + credential_mode = credential.routing_mode if credential else "inherit" + + if credential_mode == "direct": + return None, "direct" + + use_gateway = _org_wants_gateway(org, platform, credential_mode=credential_mode) + if not use_gateway: + return None, "direct" + + strict = credential_mode == "gateway" + config = _build_gateway_config( + organization_id, + org, + platform, + credential=credential, + credential_mode=credential_mode, + strict=strict, + ) + if config is None: + return None, "direct" + + return config, config.gateway_type -def _model_uses_native_provider_path(model: Any) -> bool: - model_str = str(model or "").lower() - return any(model_str.startswith(prefix) for prefix in _NATIVE_GATEWAY_MODEL_PREFIXES) +def resolve_llm_gateway( + organization_id: UUID, + db: Session, + *, + credential: Optional[CredentialRoutingContext] = None, +) -> Optional[LLMGatewayConfig]: + """Return effective LLM gateway config, or ``None`` for direct routing.""" + config, effective = resolve_effective_routing(organization_id, db, credential) + if effective == "direct": + return None + return config + + +def get_credential_effective_gateway_interface( + organization_id: UUID, + db: Session, + gateway_interface: Optional[str], +) -> GatewayInterface: + """Resolved Bifrost API surface for a credential (for API responses).""" + credential = CredentialRoutingContext( + gateway_interface=_normalize_gateway_interface(gateway_interface or "inherit"), + ) + org = _get_org_raw_settings(organization_id, db) + platform = _platform_config() + return _resolve_gateway_interface(credential, org, platform) + + +def get_credential_effective_routing_label( + organization_id: UUID, + db: Session, + routing_mode: Any, +) -> EffectiveRouting: + """Resolved routing label for API responses.""" + ctx = CredentialRoutingContext(routing_mode=_normalize_routing_mode(routing_mode)) + _, effective = resolve_effective_routing(organization_id, db, ctx) + return effective + + +# Gateways speak OpenAI-compatible ``/v1/chat/completions``. LiteLLM +# otherwise auto-detects bare names like ``gemini-2.5-flash`` as Vertex +# native and bypasses ``api_base``. def _apply_proxy_compatible_routing( @@ -257,11 +582,9 @@ def _apply_proxy_compatible_routing( *, routing_model: Optional[str] = None, ) -> Dict[str, Any]: - """Route native-path providers through the gateway's chat-completions API.""" + """Force OpenAI chat-completions routing for every gateway call.""" result = dict(call_kwargs) - model_for_routing = result.get("model") or routing_model - if _model_uses_native_provider_path(model_for_routing): - result["custom_llm_provider"] = "openai" + result["custom_llm_provider"] = "openai" return result @@ -273,6 +596,22 @@ def _strip_provider_keys(result: Dict[str, Any]) -> None: result.pop("api_key", None) +def _merge_gateway_extra_headers( + call_kwargs: Dict[str, Any], + config: LLMGatewayConfig, +) -> Dict[str, str]: + merged = dict(call_kwargs.get("extra_headers") or {}) + if config.extra_headers: + merged.update(config.extra_headers) + if config.virtual_key: + header_name = config.auth_header or "x-bf-vk" + merged[header_name] = _format_gateway_auth_header_value( + header_name, + config.virtual_key, + ) + return merged + + def _apply_bifrost_gateway( call_kwargs: Dict[str, Any], config: LLMGatewayConfig, @@ -282,10 +621,9 @@ def _apply_bifrost_gateway( result = dict(call_kwargs) result["api_base"] = config.api_base - if config.virtual_key: - extra_headers = dict(result.get("extra_headers") or {}) - extra_headers["x-bf-vk"] = config.virtual_key - result["extra_headers"] = extra_headers + merged_headers = _merge_gateway_extra_headers(result, config) + if merged_headers: + result["extra_headers"] = merged_headers if not config.passthrough_provider_keys: _strip_provider_keys(result) @@ -293,14 +631,6 @@ def _apply_bifrost_gateway( if not result.get("api_key"): result["api_key"] = config.virtual_key or LITELLM_GATEWAY_PLACEHOLDER_API_KEY - model = result.get("model") or routing_model or "" - if model and "/" not in str(model): - logger.warning( - "Routing model '{}' through Bifrost without a provider prefix; " - "ensure the model is supported by both LiteLLM and Bifrost.", - model, - ) - return _apply_proxy_compatible_routing(result, routing_model=routing_model) @@ -313,6 +643,10 @@ def _apply_litellm_proxy_gateway( result = dict(call_kwargs) result["api_base"] = config.api_base + merged_headers = _merge_gateway_extra_headers(result, config) + if merged_headers: + result["extra_headers"] = merged_headers + if not config.passthrough_provider_keys: _strip_provider_keys(result) @@ -328,6 +662,7 @@ def apply_llm_gateway( organization_id: UUID, db: Session, model: Optional[str] = None, + credential: Optional[CredentialRoutingContext] = None, ) -> Dict[str, Any]: """Merge gateway proxy settings into LiteLLM ``completion`` kwargs. @@ -335,7 +670,7 @@ def apply_llm_gateway( ``DefaultAdapter``) so native-path routing still applies without putting ``model`` in the returned kwargs. """ - config = resolve_llm_gateway(organization_id, db) + config = resolve_llm_gateway(organization_id, db, credential=credential) if config is None: return call_kwargs @@ -348,10 +683,16 @@ def litellm_completion( *, organization_id: UUID, db: Session, + credential: Optional[CredentialRoutingContext] = None, **kwargs: Any, ): """Call ``litellm.completion`` with LLM gateway settings applied.""" import litellm - kwargs = apply_llm_gateway(kwargs, organization_id=organization_id, db=db) + kwargs = apply_llm_gateway( + kwargs, + organization_id=organization_id, + db=db, + credential=credential, + ) return litellm.completion(**kwargs) diff --git a/app/services/ai/llm_gateway_settings.py b/app/services/ai/llm_gateway_settings.py index 42662dd..95f5e45 100644 --- a/app/services/ai/llm_gateway_settings.py +++ b/app/services/ai/llm_gateway_settings.py @@ -16,9 +16,11 @@ from app.models.database import Organization from app.services.ai.llm_gateway import ( EffectiveRouting, + GatewayInterface, GatewayType, _platform_config, gateway_managed_credentials_enabled, + normalize_bifrost_native_url, normalize_bifrost_url, normalize_litellm_proxy_url, resolve_llm_gateway, @@ -26,6 +28,7 @@ GatewayMode = Literal["inherit", "enabled", "disabled"] GatewayTypeOverride = Literal["inherit", "bifrost", "litellm_proxy"] +GatewayInterfaceOverride = Literal["inherit", "litellm_shim", "native_openai"] def _mode_from_enabled(enabled: Any) -> GatewayMode: @@ -57,8 +60,33 @@ def _effective_routing(resolved: Any) -> EffectiveRouting: return resolved.gateway_type -def _normalize_url_for_type(base_url: str, gateway_type: GatewayType) -> str: +def _gateway_interface_from_stored(raw: Dict[str, Any]) -> GatewayInterfaceOverride: + stored = raw.get("gateway_interface") + if stored in ("litellm_shim", "native_openai"): + return stored + return "inherit" + + +def _resolve_effective_gateway_interface( + org_raw: Dict[str, Any], platform: Dict[str, Any] +) -> GatewayInterface: + org_interface = org_raw.get("gateway_interface") + if org_interface in ("litellm_shim", "native_openai"): + return org_interface + platform_interface = platform.get("gateway_interface", "litellm_shim") + if platform_interface in ("litellm_shim", "native_openai"): + return platform_interface + return "litellm_shim" + + +def _normalize_url_for_type( + base_url: str, + gateway_type: GatewayType, + gateway_interface: GatewayInterface = "litellm_shim", +) -> str: if gateway_type == "bifrost": + if gateway_interface == "native_openai": + return normalize_bifrost_native_url(base_url) return normalize_bifrost_url(base_url) return normalize_litellm_proxy_url(base_url) @@ -85,22 +113,27 @@ def get_org_settings(organization_id: UUID, db: Session) -> Dict[str, Any]: has_virtual_key = bool(raw.get("virtual_key")) has_master_key = bool(raw.get("master_key")) gateway_type = _gateway_type_from_stored(raw) + gateway_interface = _gateway_interface_from_stored(raw) resolved = resolve_llm_gateway(organization_id, db) platform = _platform_config() effective_type = _resolve_effective_gateway_type(raw, platform) + effective_interface = _resolve_effective_gateway_interface(raw, platform) return { "mode": mode, "gateway_type": gateway_type, + "gateway_interface": gateway_interface, "base_url": stored_base_url, "has_virtual_key": has_virtual_key, "has_master_key": has_master_key, "platform_enabled": platform["enabled"], "platform_gateway_type": platform["gateway_type"], + "platform_gateway_interface": platform.get("gateway_interface", "litellm_shim"), "platform_base_url": platform["base_url"], "effective_routing": _effective_routing(resolved), "effective_gateway_type": effective_type if resolved else None, + "effective_gateway_interface": effective_interface if resolved else None, "effective_base_url": resolved.api_base if resolved else None, "effective_has_virtual_key": bool(resolved and resolved.virtual_key), "effective_has_master_key": bool(resolved and resolved.master_key), @@ -114,6 +147,7 @@ def set_org_settings( *, mode: GatewayMode, gateway_type: GatewayTypeOverride = "inherit", + gateway_interface: GatewayInterfaceOverride = "inherit", base_url: Optional[str] = None, virtual_key: Optional[str] = None, master_key: Optional[str] = None, @@ -136,17 +170,31 @@ def set_org_settings( else: payload["gateway_type"] = gateway_type + if gateway_interface == "inherit": + payload["gateway_interface"] = None + else: + payload["gateway_interface"] = gateway_interface + effective_type_for_validation: GatewayType if gateway_type in ("bifrost", "litellm_proxy"): effective_type_for_validation = gateway_type else: effective_type_for_validation = _resolve_effective_gateway_type({}, platform) + effective_interface_for_validation = _resolve_effective_gateway_interface( + {"gateway_interface": payload.get("gateway_interface")}, + platform, + ) + if base_url is not None: trimmed = base_url.strip() if trimmed: try: - _normalize_url_for_type(trimmed, effective_type_for_validation) + _normalize_url_for_type( + trimmed, + effective_type_for_validation, + effective_interface_for_validation, + ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc payload["base_url"] = trimmed diff --git a/app/services/ai/llm_resolver.py b/app/services/ai/llm_resolver.py index 1c46157..d314348 100644 --- a/app/services/ai/llm_resolver.py +++ b/app/services/ai/llm_resolver.py @@ -11,10 +11,16 @@ from uuid import UUID from fastapi import HTTPException +from sqlalchemy import desc from sqlalchemy.orm import Session from app.models.database import AIProvider from app.models.enums import ModelProvider +from app.services.ai.llm_gateway import ( + resolve_effective_routing, + routing_context_from_ai_provider, +) +from app.services.credentials import resolve_ai_provider _DEFAULT_MODELS: dict[ModelProvider, str] = { @@ -23,54 +29,165 @@ ModelProvider.GOOGLE: "gemini-2.0-flash", } +_AUTO_DETECT_PRIORITY = ( + ModelProvider.OPENAI, + ModelProvider.ANTHROPIC, + ModelProvider.GOOGLE, +) + + +def _provider_enum(provider: str) -> ModelProvider: + try: + return ModelProvider(provider.lower()) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unsupported LLM provider: {provider}", + ) + + +def _default_model_for(provider: ModelProvider) -> str: + return _DEFAULT_MODELS.get(provider, "gpt-5-mini") + + +def _resolved_model_for_row( + organization_id: UUID, + db: Session, + ai_prov: AIProvider, + explicit_model: Optional[str], +) -> str: + if explicit_model: + return explicit_model + ctx = routing_context_from_ai_provider(ai_prov) + _, effective = resolve_effective_routing(organization_id, db, ctx) + if effective != "direct" and ctx.gateway_model: + return ctx.gateway_model + try: + return _default_model_for(ModelProvider(ai_prov.provider.lower())) + except ValueError: + return "gpt-5-mini" + def get_llm_provider_and_model( organization_id: UUID, db: Session, provider: Optional[str] = None, model: Optional[str] = None, + credential_id: Optional[UUID] = None, ) -> Tuple[ModelProvider, str]: """Resolve ``(provider_enum, model_str)`` for a one-off LLM call. - * If both ``provider`` and ``model`` are supplied by the caller we - validate the provider against the ``ModelProvider`` enum and pass - both straight through. - * Otherwise we look at the org's active ``AIProvider`` rows in the - preference order ``OpenAI -> Anthropic -> Google`` and pair the - first match with a sensible default model. + * When ``credential_id`` is supplied we resolve that exact active + ``AIProvider`` row (required when multiple credentials share a + provider, e.g. several ``custom`` gateway models). + * When ``provider`` is supplied we resolve the matching active + ``AIProvider`` row. An omitted ``model`` uses the credential's + ``gateway_model`` when gateway routing is active, otherwise a + provider-specific default. + * When both are omitted we prefer the org-wide default credential, + then OpenAI -> Anthropic -> Google, then any other active row. * Raises ``HTTPException(400)`` with an actionable message when no - AI provider has been configured at all. + AI provider has been configured. Tests rely on patching ``app.services.ai.llm_resolver.get_llm_provider_and_model``, so the public surface here is intentionally minimal. """ - if provider and model: - try: - provider_enum = ModelProvider(provider.lower()) - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Unsupported LLM provider: {provider}", - ) - return provider_enum, model + explicit_model = (model or "").strip() or None + explicit_provider = (provider or "").strip() or None - for prov in (ModelProvider.OPENAI, ModelProvider.ANTHROPIC, ModelProvider.GOOGLE): + if credential_id is not None: ai_prov = ( db.query(AIProvider) .filter( + AIProvider.id == credential_id, AIProvider.organization_id == organization_id, - AIProvider.is_active == True, # noqa: E712 (SQLAlchemy boolean) - AIProvider.provider == prov.value, + AIProvider.is_active == True, # noqa: E712 ) .first() ) + if not ai_prov: + raise HTTPException( + status_code=400, + detail=( + f"No active AI provider found for credential {credential_id}. " + "Check AI Providers settings." + ), + ) + provider_enum = _provider_enum(ai_prov.provider) + if ( + explicit_provider + and explicit_provider.lower() != ai_prov.provider.lower() + ): + raise HTTPException( + status_code=400, + detail="provider does not match the selected credential.", + ) + model_str = _resolved_model_for_row( + organization_id, db, ai_prov, explicit_model + ) + return provider_enum, model_str + + if explicit_provider: + provider_enum = _provider_enum(explicit_provider) + ai_prov = resolve_ai_provider(provider_enum.value, db, organization_id) + if not ai_prov: + raise HTTPException( + status_code=400, + detail=( + f"No active AI provider configured for {explicit_provider}. " + "Add one in AI Providers settings." + ), + ) + model_str = _resolved_model_for_row( + organization_id, db, ai_prov, explicit_model + ) + return provider_enum, model_str + + default_row = ( + db.query(AIProvider) + .filter( + AIProvider.organization_id == organization_id, + AIProvider.is_active == True, # noqa: E712 (SQLAlchemy boolean) + AIProvider.is_default == True, # noqa: E712 + ) + .order_by(desc(AIProvider.updated_at)) + .first() + ) + if default_row: + provider_enum = _provider_enum(default_row.provider) + model_str = _resolved_model_for_row( + organization_id, db, default_row, explicit_model + ) + return provider_enum, model_str + + for prov in _AUTO_DETECT_PRIORITY: + ai_prov = resolve_ai_provider(prov.value, db, organization_id) if ai_prov: - return prov, model or _DEFAULT_MODELS.get(prov, "gpt-5-mini") + model_str = _resolved_model_for_row( + organization_id, db, ai_prov, explicit_model + ) + return prov, model_str + + fallback = ( + db.query(AIProvider) + .filter( + AIProvider.organization_id == organization_id, + AIProvider.is_active == True, # noqa: E712 + ) + .order_by(desc(AIProvider.updated_at)) + .first() + ) + if fallback: + provider_enum = _provider_enum(fallback.provider) + model_str = _resolved_model_for_row( + organization_id, db, fallback, explicit_model + ) + return provider_enum, model_str raise HTTPException( status_code=400, detail=( - "No active AI provider configured. Add an OpenAI, Anthropic, " - "or Google provider in AI Providers settings." + "No active AI provider configured. Add an AI provider " + "in AI Providers settings." ), ) diff --git a/app/services/ai/llm_service.py b/app/services/ai/llm_service.py index e705a6c..0fde20a 100644 --- a/app/services/ai/llm_service.py +++ b/app/services/ai/llm_service.py @@ -20,7 +20,13 @@ from app.models.database import ModelProvider, AIProvider from app.services.credentials import resolve_ai_provider, resolve_integration from app.services.ai.llm_generation_config import build_litellm_kwargs -from app.services.ai.llm_gateway import apply_llm_gateway, resolve_litellm_api_key +from app.services.ai.llm_gateway import ( + apply_llm_gateway, + resolve_effective_routing, + resolve_litellm_api_key, + resolve_litellm_model, + routing_context_from_ai_provider, +) # LiteLLM will silently drop params the target provider doesn't support # rather than raising an error. @@ -250,15 +256,31 @@ def generate_response( ai_provider = self._get_ai_provider( llm_provider, db, organization_id, credential_id=credential_id ) + credential_ctx = ( + routing_context_from_ai_provider(ai_provider) if ai_provider else None + ) if ai_provider: - api_key = resolve_litellm_api_key(organization_id, db, ai_provider) + api_key = resolve_litellm_api_key( + organization_id, + db, + ai_provider, + credential=credential_ctx, + ) else: api_key = self._resolve_api_key( llm_provider, db, organization_id, credential_id=credential_id ) # --- call LiteLLM -------------------------------------------------- - model_str = self._litellm_model_name(llm_provider, llm_model) + workload_model_str = self._litellm_model_name(llm_provider, llm_model) + _, effective_routing = resolve_effective_routing( + organization_id, db, credential_ctx + ) + model_str = resolve_litellm_model( + workload_model_str=workload_model_str, + gateway_active=effective_routing != "direct", + credential=credential_ctx, + ) call_kwargs: Dict[str, Any] = { "model": model_str, @@ -302,6 +324,8 @@ def generate_response( call_kwargs, organization_id=organization_id, db=db, + model=model_str, + credential=credential_ctx, ) try: diff --git a/app/services/ai/stt_clients/google.py b/app/services/ai/stt_clients/google.py index 7e33581..6d86ec3 100644 --- a/app/services/ai/stt_clients/google.py +++ b/app/services/ai/stt_clients/google.py @@ -91,6 +91,7 @@ def transcribe_google( *, organization_id: Optional["UUID"] = None, db: Optional[Any] = None, + credential: Optional[Any] = None, ) -> Dict[str, Any]: """Transcribe an audio file via Gemini through LiteLLM. @@ -140,6 +141,7 @@ def transcribe_google( call_kwargs, organization_id=organization_id, db=db, + credential=credential, ) response = litellm.completion(**call_kwargs) except Exception as e: diff --git a/app/services/ai/transcription_service.py b/app/services/ai/transcription_service.py index c5263db..fbe87c7 100644 --- a/app/services/ai/transcription_service.py +++ b/app/services/ai/transcription_service.py @@ -90,13 +90,22 @@ def _get_api_key_for_provider( preferred in either table; otherwise the row marked ``is_default`` wins, with a back-compat fallback to the most recent active row. """ - from app.services.ai.llm_gateway import resolve_litellm_api_key + from app.services.ai.llm_gateway import ( + resolve_litellm_api_key, + routing_context_from_ai_provider, + ) ai_provider = self._get_ai_provider( provider, db, organization_id, credential_id=credential_id ) if ai_provider: - return resolve_litellm_api_key(organization_id, db, ai_provider) + credential_ctx = routing_context_from_ai_provider(ai_provider) + return resolve_litellm_api_key( + organization_id, + db, + ai_provider, + credential=credential_ctx, + ) integration = resolve_integration( provider, db, organization_id, credential_id=credential_id @@ -106,6 +115,22 @@ def _get_api_key_for_provider( return None + def _get_credential_context_for_provider( + self, + provider: ModelProvider, + db: Session, + organization_id: UUID, + credential_id: Optional[UUID] = None, + ): + from app.services.ai.llm_gateway import routing_context_from_ai_provider + + ai_provider = self._get_ai_provider( + provider, db, organization_id, credential_id=credential_id + ) + if ai_provider: + return routing_context_from_ai_provider(ai_provider) + return None + def _download_audio_to_temp(self, audio_file_key: str, db: Optional[Session] = None) -> str: """ Download audio from S3 to temporary file, or use local file if S3 is not available. @@ -515,6 +540,14 @@ def transcribe_text_only( ) return None + credential_ctx = ( + self._get_credential_context_for_provider( + stt_provider, db, organization_id, credential_id=credential_id + ) + if stt_provider == ModelProvider.GOOGLE + else None + ) + from app.services.ai.stt_clients import ( transcribe_openai, transcribe_deepgram, @@ -535,6 +568,7 @@ def transcribe_text_only( result = transcribe_google( audio_file_path, stt_model, api_key, language, organization_id=organization_id, db=db, + credential=credential_ctx, ) elif stt_provider == ModelProvider.SARVAM: result = transcribe_sarvam(audio_file_path, stt_model, api_key, language) @@ -574,6 +608,9 @@ def transcribe( api_key = self._get_api_key_for_provider( stt_provider, db, organization_id, credential_id=credential_id ) + credential_ctx = self._get_credential_context_for_provider( + stt_provider, db, organization_id, credential_id=credential_id + ) if not api_key and stt_provider != ModelProvider.GOOGLE: raise RuntimeError( f"No API key found for {stt_provider} (checked AIProvider and Integration tables). " @@ -615,6 +652,7 @@ def transcribe( result = transcribe_google( temp_file_path, stt_model, api_key, language, organization_id=organization_id, db=db, + credential=credential_ctx, ) elif stt_provider == ModelProvider.AZURE: raise NotImplementedError("Azure Speech Services not yet implemented") diff --git a/app/services/judge_alignment/gepa_bridge.py b/app/services/judge_alignment/gepa_bridge.py index c6fe969..37926fa 100644 --- a/app/services/judge_alignment/gepa_bridge.py +++ b/app/services/judge_alignment/gepa_bridge.py @@ -28,7 +28,15 @@ from loguru import logger from sqlalchemy.orm import Session -from app.services.ai.llm_gateway import apply_llm_gateway, litellm_completion, resolve_litellm_api_key +from app.services.ai.llm_gateway import ( + apply_llm_gateway, + CredentialRoutingContext, + litellm_completion, + resolve_effective_routing, + resolve_litellm_api_key, + resolve_litellm_model, + routing_context_from_ai_provider, +) from app.services.credentials import resolve_ai_provider from app.models.database import ( @@ -57,13 +65,32 @@ def _resolve_litellm_model(provider: str, model: str) -> str: return LLMService._litellm_model_name(provider, model) # type: ignore[arg-type] -def _resolve_api_key( - provider: str, organization_id: UUID, db: Session -) -> Optional[str]: +def _resolve_lm_call( + provider: str, + model: str, + organization_id: UUID, + db: Session, +) -> tuple[str, Optional[str], Optional[CredentialRoutingContext]]: ai_provider = resolve_ai_provider(provider, db, organization_id) if not ai_provider: raise RuntimeError(f"No active AIProvider for {provider!r}") - return resolve_litellm_api_key(organization_id, db, ai_provider) + credential_ctx = routing_context_from_ai_provider(ai_provider) + workload_model_str = _resolve_litellm_model(provider, model) + _, effective_routing = resolve_effective_routing( + organization_id, db, credential_ctx + ) + model_str = resolve_litellm_model( + workload_model_str=workload_model_str, + gateway_active=effective_routing != "direct", + credential=credential_ctx, + ) + api_key = resolve_litellm_api_key( + organization_id, + db, + ai_provider, + credential=credential_ctx, + ) + return model_str, api_key, credential_ctx def start_gepa_for_dataset( @@ -216,8 +243,12 @@ def execute_judge_gepa(run_id: str, db: Session) -> Dict[str, Any]: dataset_output_field = judge_dataset.output_field if judge_dataset else None provider_value = (evaluator.llm_provider or "").lower() - api_key = _resolve_api_key(provider_value, run.organization_id, db) - lm_identifier = _resolve_litellm_model(provider_value, evaluator.llm_model) + lm_identifier, api_key, credential_ctx = _resolve_lm_call( + provider_value, + evaluator.llm_model, + run.organization_id, + db, + ) gepa_optimize, DefaultAdapter = _ensure_gepa() @@ -249,7 +280,7 @@ def _evaluate_prompt(candidate_prompt: str) -> float: } if api_key is not None: sample_kwargs["api_key"] = api_key - resp = litellm_completion(**sample_kwargs) + resp = litellm_completion(**sample_kwargs, credential=credential_ctx) text = resp.choices[0].message.content if resp.choices else "" parsed = _parse_judge_response(text) except Exception as exc: @@ -304,6 +335,7 @@ def evaluator_fn(data: Dict[str, Any], response: str) -> Any: organization_id=run.organization_id, db=db, model=lm_identifier, + credential=credential_ctx, ) adapter = DefaultAdapter( @@ -335,7 +367,7 @@ def reflection_lm(prompt: str) -> str: } if api_key is not None: reflection_kwargs["api_key"] = api_key - resp = litellm_completion(**reflection_kwargs) + resp = litellm_completion(**reflection_kwargs, credential=credential_ctx) return resp.choices[0].message.content run.status = PromptOptimizationStatus.RUNNING.value diff --git a/app/services/optimization/gepa_service.py b/app/services/optimization/gepa_service.py index fe6bf2b..5328dba 100644 --- a/app/services/optimization/gepa_service.py +++ b/app/services/optimization/gepa_service.py @@ -21,7 +21,7 @@ ) from app.services.optimization.data_preparation import build_trainset from app.services.optimization.evaluator import build_evaluator -from app.services.optimization.lm_resolver import resolve_api_key, resolve_lm +from app.services.optimization.lm_resolver import resolve_lm_call from app.services.ai.llm_gateway import apply_llm_gateway, litellm_completion _gepa_install_attempted = False @@ -95,8 +95,13 @@ def run_optimization( minibatch_size = config.get("minibatch_size", 5) seed_prompt = agent.provider_prompt or agent.description or "" - lm_identifier = resolve_lm(voice_bundle, evaluator) - api_key = resolve_api_key(lm_identifier, ai_providers, organization_id, db) + lm_identifier, api_key, credential_ctx = resolve_lm_call( + voice_bundle, + evaluator, + ai_providers, + organization_id, + db, + ) trainset = build_trainset(training_data, metrics) if not trainset: @@ -123,6 +128,7 @@ def run_optimization( organization_id=organization_id, db=db, model=lm_identifier, + credential=credential_ctx, ) adapter = DefaultAdapter( @@ -140,7 +146,7 @@ def reflection_lm(prompt: str) -> str: } if api_key is not None: reflection_kwargs["api_key"] = api_key - resp = litellm_completion(**reflection_kwargs) + resp = litellm_completion(**reflection_kwargs, credential=credential_ctx) return resp.choices[0].message.content result = gepa_optimize( diff --git a/app/services/optimization/lm_resolver.py b/app/services/optimization/lm_resolver.py index 18acaa8..d41a578 100644 --- a/app/services/optimization/lm_resolver.py +++ b/app/services/optimization/lm_resolver.py @@ -6,13 +6,19 @@ passed explicitly on every LiteLLM call -- no environment variables mutated. """ -from typing import List, Optional +from typing import List, Optional, Tuple from uuid import UUID from sqlalchemy.orm import Session from app.models.database import AIProvider, Evaluator, VoiceBundle -from app.services.ai.llm_gateway import resolve_litellm_api_key +from app.services.ai.llm_gateway import ( + CredentialRoutingContext, + resolve_effective_routing, + resolve_litellm_api_key, + resolve_litellm_model, + routing_context_from_ai_provider, +) def resolve_lm( @@ -30,23 +36,97 @@ def resolve_lm( return "openai/gpt-4o" +def _find_ai_provider_for_lm( + lm_identifier: str, + ai_providers: List[AIProvider], + *, + voice_bundle: Optional[VoiceBundle] = None, +) -> Optional[AIProvider]: + if voice_bundle and voice_bundle.llm_credential_id: + for provider in ai_providers: + if provider.id == voice_bundle.llm_credential_id and provider.is_active: + return provider + + provider_prefix = lm_identifier.split("/")[0].lower() + fallback: Optional[AIProvider] = None + for provider in ai_providers: + if not provider.is_active or not provider.api_key: + continue + if provider.provider.lower() != provider_prefix: + continue + if provider.is_default: + return provider + if fallback is None: + fallback = provider + return fallback + + def resolve_api_key( lm_identifier: str, ai_providers: List[AIProvider], organization_id: UUID, db: Session, + *, + voice_bundle: Optional[VoiceBundle] = None, + credential: Optional[CredentialRoutingContext] = None, ) -> Optional[str]: """ Given ``"openai/gpt-5.4"`` and the org's provider list, decrypt and return the matching API key, or None when Bifrost manages provider keys. """ - provider_prefix = lm_identifier.split("/")[0].lower() - for p in ai_providers: - if not p.is_active or not p.api_key: - continue - if p.provider.lower() == provider_prefix: - return resolve_litellm_api_key(organization_id, db, p) - raise RuntimeError( - f"No active AI provider matching '{provider_prefix}' found. " - "Add one in Settings > AI Providers." + ai_provider = _find_ai_provider_for_lm( + lm_identifier, + ai_providers, + voice_bundle=voice_bundle, + ) + if not ai_provider: + raise RuntimeError( + f"No active AI provider matching '{lm_identifier.split('/')[0].lower()}' found. " + "Add one in Settings > AI Providers." + ) + ctx = credential or routing_context_from_ai_provider(ai_provider) + return resolve_litellm_api_key( + organization_id, + db, + ai_provider, + credential=ctx, + ) + + +def resolve_lm_call( + voice_bundle: Optional[VoiceBundle], + evaluator: Optional[Evaluator], + ai_providers: List[AIProvider], + organization_id: UUID, + db: Session, +) -> Tuple[str, Optional[str], Optional[CredentialRoutingContext]]: + """Resolve model string, API key, and credential routing context for GEPA.""" + lm_identifier = resolve_lm(voice_bundle, evaluator) + ai_provider = _find_ai_provider_for_lm( + lm_identifier, + ai_providers, + voice_bundle=voice_bundle, + ) + if not ai_provider: + raise RuntimeError( + f"No active AI provider matching '{lm_identifier.split('/')[0].lower()}' found. " + "Add one in Settings > AI Providers." + ) + credential_ctx = routing_context_from_ai_provider(ai_provider) + _, effective_routing = resolve_effective_routing( + organization_id, db, credential_ctx + ) + model_str = resolve_litellm_model( + workload_model_str=lm_identifier, + gateway_active=effective_routing != "direct", + credential=credential_ctx, + ) + api_key = resolve_api_key( + lm_identifier, + ai_providers, + organization_id, + db, + voice_bundle=voice_bundle, + credential=credential_ctx, ) + return model_str, api_key, credential_ctx diff --git a/config.yml.example b/config.yml.example index ccc22d0..ddd746a 100644 --- a/config.yml.example +++ b/config.yml.example @@ -175,6 +175,7 @@ judge_alignment: # virtual_key: null # Bifrost only (x-bf-vk header) # master_key: null # LiteLLM Proxy only (LITELLM_MASTER_KEY) # passthrough_provider_keys: true # false = keys in gateway; AI integrations can omit api_key +# gateway_interface: litellm_shim # litellm_shim | native_openai (Bifrost only) # Enterprise License (JWT signed with RS256). Unlocks gated features like # oidc_sso, mfa_enforce, audit_export, voice_playground, gepa_optimization, diff --git a/docs-fumadocs/content/docs/getting-started/llm-gateway.mdx b/docs-fumadocs/content/docs/getting-started/llm-gateway.mdx index 60a77c4..47f11bd 100644 --- a/docs-fumadocs/content/docs/getting-started/llm-gateway.mdx +++ b/docs-fumadocs/content/docs/getting-started/llm-gateway.mdx @@ -14,6 +14,7 @@ Supported gateway types: |------|-------------| | **Direct** | Default — LiteLLM calls providers using org AI Provider credentials | | **Bifrost** | [Bifrost](https://docs.getbifrost.ai/) `/litellm` proxy with optional `x-bf-vk` virtual key | +| **Bifrost (native)** | Bifrost root OpenAI-compatible API (`/v1/chat/completions`) without the `/litellm` shim | | **LiteLLM Proxy** | Self-hosted [LiteLLM Proxy](https://docs.litellm.ai/docs/proxy/quick_start) with optional master key | ## What is routed through the gateway @@ -39,16 +40,18 @@ llm_gateway: virtual_key: null # Bifrost only (x-bf-vk header) master_key: null # LiteLLM Proxy only (LITELLM_MASTER_KEY) passthrough_provider_keys: false + gateway_interface: litellm_shim # litellm_shim | native_openai (Bifrost only) ``` | Setting | Description | |---------|-------------| | `enabled` | Platform-wide toggle for orgs that inherit the default | | `type` | Gateway implementation: `bifrost` or `litellm_proxy` | -| `base_url` | Gateway endpoint (Bifrost: must end with `/litellm`; LiteLLM Proxy: e.g. `http://localhost:4000`) | +| `base_url` | Gateway endpoint (Bifrost shim: include `/litellm`; native: host root e.g. `http://localhost:8080` — EfficientAI appends `/v1` for LiteLLM; LiteLLM Proxy: e.g. `http://localhost:4000`) | | `virtual_key` | Optional Bifrost `x-bf-vk` header | | `master_key` | Optional LiteLLM Proxy master key | | `passthrough_provider_keys` | When `true`, org AI provider API keys are still sent through the gateway | +| `gateway_interface` | Bifrost API surface: `litellm_shim` (default) or `native_openai` for custom models that fail through `/litellm` | When routing through a gateway, Gemini models automatically use OpenAI-compatible proxy endpoints (`/v1/chat/completions`) instead of native Gemini `:generateContent` URLs. This is required for Bifrost and LiteLLM Proxy compatibility. @@ -76,6 +79,58 @@ On **Configurations → Integrations**, open **LLM Gateway** to configure per-or Org virtual keys and LiteLLM Proxy master keys are encrypted at rest like AI provider credentials. +## Per-integration routing (AI Providers and Voice Platforms) + +Each **AI Provider** and **Voice Platform** credential can override org-level gateway behavior: + +| Routing mode | Behavior | +|--------------|----------| +| **Inherit org default** | Follow org/platform LLM gateway settings | +| **Route via gateway** | Force batch/eval LiteLLM calls through the gateway, even if the org opted out | +| **Direct API key** | Always call the provider directly, even when the org gateway is enabled | + +### Bifrost custom models (AI Providers) + +When creating or editing an AI Provider with gateway routing, you can set an optional **Gateway Model** — a Bifrost custom model ID (e.g. `production-gpt4` or `openai/gpt-4o`). When set, batch/eval workloads that use this credential send that model string to the gateway instead of the workload-selected model. + +You can also override **Bifrost API surface** per credential: + +| Interface | `api_base` | Use case | +|-----------|------------|----------| +| **LiteLLM shim** (default) | `http://bifrost:8080/litellm` | Standard models through Bifrost's LiteLLM integration | +| **Native OpenAI-compatible** | `http://bifrost:8080` (no `/litellm`) | Custom models (e.g. Gemma on customer cloud) that only work on Bifrost's native API | + +Optional **Gateway Base URL** on the credential overrides the org/platform URL for that integration only. + +### Per-credential gateway auth (no org virtual key required) + +Each AI Provider can override Bifrost authentication independently of org LLM Gateway settings: + +| Field | Maps to | Description | +|-------|---------|-------------| +| **Gateway Model** | `model` | Bifrost model ID, e.g. `inhouse-llm-server-v2//models/gemma-4-31B-it-AWQ-4bit` | +| **Gateway Base URL** | `endpoint` | Per-credential Bifrost host, e.g. `http://localhost:8080` | +| **Gateway Auth Header** | `authHeaderName` | Header name (default `x-bf-vk`; also supports `Authorization`, `x-api-key`, etc.) | +| **Gateway Auth Secret Env Var** | `authSecretName` | Runtime env var holding the virtual key, e.g. `BIFROST_PRD_VK_GEMMA_4_31B` | +| **Gateway Auth Secret** | inline secret | Alternative to env var — encrypted at rest in the DB | +| **Gateway Extra Headers** | arbitrary headers | JSON object of additional HTTP headers, e.g. `{"X-Custom-Tenant": "prod"}` | + +Auth secret resolution order: credential env var → credential inline secret → org virtual key → platform virtual key. Configured auth header wins over the same key in **Gateway Extra Headers**. + +**Example — custom Gemma credential via Bifrost (no org VK configured):** + +1. AI Provider: `routing_mode=gateway`, `gateway_interface=native_openai` +2. Gateway Model: `inhouse-llm-server-v2//models/gemma-4-31B-it-AWQ-4bit` +3. Gateway Base URL: `http://localhost:8080` +4. Gateway Auth Header: `x-bf-vk` +5. Gateway Auth Secret Env Var: `BIFROST_PRD_VK_GEMMA_4_31B` (mounted in the EfficientAI deployment) + +Workload UIs (evaluators, voice bundles, call import) continue to pick provider + model + credential as before. The selected credential controls routing, API surface, URL override, auth header/secret, and optional gateway model override. + +### Voice platform limitations + +Voice platform integrations store routing preferences for consistency, but **real-time voice agents always use direct API keys**. Retell, Vapi, and native STT/TTS SDK paths are not routed through the LLM gateway today. + ## Provider compatibility Only providers supported by **both** LiteLLM and your gateway deployment work through the proxy. EfficientAI's LiteLLM integration covers OpenAI, Anthropic, Google/Gemini, Azure, AWS Bedrock, DeepSeek, Groq, xAI, and Fireworks. @@ -87,7 +142,7 @@ Only providers supported by **both** LiteLLM and your gateway deployment work th ## Example call shapes -**Bifrost:** +**Bifrost (LiteLLM shim):** ```python litellm.completion( @@ -98,6 +153,17 @@ litellm.completion( ) ``` +**Bifrost (native OpenAI-compatible):** + +```python +litellm.completion( + model="custom-gemma-model", + messages=[{"role": "user", "content": "Hello!"}], + api_base="http://localhost:8080", + extra_headers={"x-bf-vk": "your-virtual-key"}, +) +``` + **LiteLLM Proxy:** ```python diff --git a/frontend/src/components/AIProviderModelPicker.tsx b/frontend/src/components/AIProviderModelPicker.tsx index 37d6439..5fcad25 100644 --- a/frontend/src/components/AIProviderModelPicker.tsx +++ b/frontend/src/components/AIProviderModelPicker.tsx @@ -1,155 +1,200 @@ -import { useEffect } from 'react' -import { useQuery } from '@tanstack/react-query' -import { Bot } from 'lucide-react' -import { apiClient } from '../lib/api' -import LLMAdvancedOptionsPanel from './providers/LLMAdvancedOptionsPanel' -import type { LLMGenerationConfig } from '../config/llmGenerationParams' - -interface AIProviderRow { - id: string - provider: string - name: string | null - is_active: boolean -} - -const PROVIDER_LABELS: Record = { - openai: 'OpenAI', - anthropic: 'Anthropic', - google: 'Google', - deepseek: 'DeepSeek', - groq: 'Groq', -} - -/** - * Inline two-up provider + model dropdown shared by every AI-generate - * surface (Prompt Partials, Visualizations TLDR, etc.). - * - * Empty string for ``provider`` means "auto-detect" -- the backend - * resolver will pick the org's first active OpenAI/Anthropic/Google - * credential. When ``provider`` is set we surface that provider's - * available LLM models from ``apiClient.getModelOptions``. - * - * The component is fully controlled: the parent owns ``provider`` / - * ``model`` state and reacts to ``onProviderChange`` / ``onModelChange`` - * so it can persist or seed the picker between mounts (e.g. seed to - * the previously-used provider on Regenerate). - */ -export default function AIProviderModelPicker({ - provider, - model, - onProviderChange, - onModelChange, - llm_config, - onLLMConfigChange, - disabled = false, - size = 'md', - showAdvancedOptions = true, -}: { - provider: string - model: string - onProviderChange: (next: string) => void - onModelChange: (next: string) => void - llm_config?: LLMGenerationConfig | null - onLLMConfigChange?: (next: LLMGenerationConfig | null) => void - disabled?: boolean - size?: 'sm' | 'md' - showAdvancedOptions?: boolean -}) { - const { data: aiProviders = [] } = useQuery({ - queryKey: ['ai-providers'], - queryFn: () => apiClient.listAIProviders(), - }) - - const activeProviders = aiProviders.filter((p) => p.is_active) - - const { data: modelOptions } = useQuery({ - queryKey: ['model-options', provider], - queryFn: () => apiClient.getModelOptions(provider), - enabled: !!provider, - }) - - const llmModels: string[] = modelOptions?.llm ?? [] - - // When the provider switches, snap the model selection to that - // provider's first available LLM so the parent never holds a model - // that doesn't belong to the active provider. - useEffect(() => { - if (provider && llmModels.length > 0 && !llmModels.includes(model)) { - onModelChange(llmModels[0]) - } - }, [provider, llmModels, model, onModelChange]) - - const inputClass = - size === 'sm' - ? 'w-full px-2.5 py-1.5 text-xs border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-primary-500 bg-white disabled:bg-gray-50 disabled:text-gray-400' - : 'w-full px-3 py-2 text-sm border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-primary-500 bg-white disabled:bg-gray-50 disabled:text-gray-400' - - const labelClass = - size === 'sm' - ? 'block text-[10px] font-medium text-gray-600 mb-1' - : 'block text-xs font-medium text-gray-600 mb-1' - - return ( -
-
-
- - - {activeProviders.length === 0 && ( -

- No AI providers configured. Add one in AI Providers settings. -

- )} -
-
- - -
-
- {showAdvancedOptions && provider && onLLMConfigChange && ( - - )} -
- ) -} +import { useEffect, useMemo } from 'react' +import { useQuery } from '@tanstack/react-query' +import { Bot } from 'lucide-react' +import { apiClient } from '../lib/api' +import LLMAdvancedOptionsPanel from './providers/LLMAdvancedOptionsPanel' +import type { LLMGenerationConfig } from '../config/llmGenerationParams' +import type { AIProvider } from '../types/api' +import { + resolveActiveAIProvider, + usesGatewayDirectModel, +} from '../lib/gatewayRouting' + +const PROVIDER_LABELS: Record = { + openai: 'OpenAI', + anthropic: 'Anthropic', + google: 'Google', + deepseek: 'DeepSeek', + groq: 'Groq', + custom: 'Custom', +} + +/** + * Inline provider + model dropdown shared by AI-generate surfaces. + * Each credential row is selectable by id so multiple integrations + * for the same provider (e.g. several custom Bifrost models) resolve + * to the intended gateway_model. + */ +export default function AIProviderModelPicker({ + provider, + model, + credentialId, + onProviderChange, + onModelChange, + onCredentialIdChange, + llm_config, + onLLMConfigChange, + disabled = false, + size = 'md', + showAdvancedOptions = true, +}: { + provider: string + model: string + credentialId?: string + onProviderChange: (next: string) => void + onModelChange: (next: string) => void + onCredentialIdChange?: (next: string) => void + llm_config?: LLMGenerationConfig | null + onLLMConfigChange?: (next: LLMGenerationConfig | null) => void + disabled?: boolean + size?: 'sm' | 'md' + showAdvancedOptions?: boolean +}) { + const { data: aiProviders = [] } = useQuery({ + queryKey: ['ai-providers'], + queryFn: () => apiClient.listAIProviders(), + }) + + const activeProviders = aiProviders.filter((p) => p.is_active) + + const selectedCredential = useMemo(() => { + if (credentialId) { + return activeProviders.find((p) => p.id === credentialId) + } + if (provider) { + return resolveActiveAIProvider(aiProviders, provider) + } + return undefined + }, [activeProviders, aiProviders, credentialId, provider]) + + const selectedCredentialId = + credentialId || selectedCredential?.id || '' + + const gatewayDirectModel = usesGatewayDirectModel(selectedCredential) + ? selectedCredential?.gateway_model?.trim() + : null + + const resolvedProvider = selectedCredential?.provider || provider + + const { data: modelOptions } = useQuery({ + queryKey: ['model-options', resolvedProvider], + queryFn: () => apiClient.getModelOptions(resolvedProvider), + enabled: !!resolvedProvider && !gatewayDirectModel, + }) + + const llmModels: string[] = modelOptions?.llm ?? [] + + useEffect(() => { + if (gatewayDirectModel) { + if (model) onModelChange('') + return + } + if (resolvedProvider && llmModels.length > 0 && !llmModels.includes(model)) { + onModelChange(llmModels[0]) + } + }, [resolvedProvider, llmModels, model, onModelChange, gatewayDirectModel]) + + const inputClass = + size === 'sm' + ? 'w-full px-2.5 py-1.5 text-xs border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-primary-500 bg-white disabled:bg-gray-50 disabled:text-gray-400' + : 'w-full px-3 py-2 text-sm border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-primary-500 bg-white disabled:bg-gray-50 disabled:text-gray-400' + + const labelClass = + size === 'sm' + ? 'block text-[10px] font-medium text-gray-600 mb-1' + : 'block text-xs font-medium text-gray-600 mb-1' + + const handleCredentialChange = (nextId: string) => { + if (!nextId) { + onCredentialIdChange?.('') + onProviderChange('') + onModelChange('') + return + } + const row = activeProviders.find((p) => p.id === nextId) + if (!row) return + onCredentialIdChange?.(row.id) + onProviderChange(row.provider) + onModelChange('') + } + + return ( +
+
+
+ + + {activeProviders.length === 0 && ( +

+ No AI providers configured. Add one in AI Providers settings. +

+ )} +
+
+ {gatewayDirectModel ? ( + <> + +
+ {gatewayDirectModel} +
+

+ Model is fixed on the integration — Bifrost gateway routing applies. +

+ + ) : ( + <> + + + + )} +
+
+ {showAdvancedOptions && resolvedProvider && !gatewayDirectModel && onLLMConfigChange && ( + + )} +
+ ) +} + \ No newline at end of file diff --git a/frontend/src/components/providers/ProviderModelPicker.tsx b/frontend/src/components/providers/ProviderModelPicker.tsx index 8e36428..810b4a0 100644 --- a/frontend/src/components/providers/ProviderModelPicker.tsx +++ b/frontend/src/components/providers/ProviderModelPicker.tsx @@ -23,14 +23,18 @@ * can implement an "Auto" / "Use run default" affordance by clearing * both fields. */ -import { useEffect } from 'react' +import { useEffect, useMemo } from 'react' import { useQuery } from '@tanstack/react-query' import { Bot, AudioLines } from 'lucide-react' import { apiClient } from '../../lib/api' -import type { Integration } from '../../types/api' +import type { AIProvider, Integration } from '../../types/api' import LLMAdvancedOptionsPanel from './LLMAdvancedOptionsPanel' import type { LLMGenerationConfig } from '../../config/llmGenerationParams' +import { + resolveActiveAIProvider, + usesGatewayDirectModel, +} from '../../lib/gatewayRouting' const PROVIDER_LABELS: Record = { openai: 'OpenAI', @@ -59,14 +63,6 @@ export interface ProviderModelValue { llm_config?: LLMGenerationConfig | null } -interface AIProviderRow { - id: string - provider: string - is_active: boolean - is_default?: boolean - name?: string | null -} - /** Origin of a credential row, used for de-duplication + tooltip copy. */ type CredentialSource = 'aiprovider' | 'integration' @@ -154,9 +150,9 @@ export default function ProviderModelPicker({ audioCapableOnly = false, showAdvancedOptions = true, }: ProviderModelPickerProps) { - const { data: aiProviders = [] } = useQuery({ + const { data: aiProviders = [] } = useQuery({ queryKey: ['ai-providers'], - queryFn: () => apiClient.listAIProviders() as Promise, + queryFn: () => apiClient.listAIProviders(), }) // STT credentials (Deepgram, Sarvam, Smallest, ElevenLabs, …) are // typically stored in the Integration table when the user adds them @@ -226,10 +222,24 @@ export default function ProviderModelPicker({ new Set(eligibleProviders.map((p) => p.provider)), ) + const activeCredential = useMemo(() => { + if (kind !== 'llm' || !value.provider) return undefined + return resolveActiveAIProvider( + aiProviders, + value.provider, + value.credential_id, + ) + }, [aiProviders, kind, value.provider, value.credential_id]) + + const gatewayDirectModel = + kind === 'llm' && usesGatewayDirectModel(activeCredential) + ? activeCredential?.gateway_model?.trim() + : null + const { data: modelOptions } = useQuery({ queryKey: ['model-options', value.provider], queryFn: () => apiClient.getModelOptions(value.provider as string), - enabled: !!value.provider, + enabled: !!value.provider && !gatewayDirectModel, }) const rawModels = @@ -246,11 +256,15 @@ export default function ProviderModelPicker({ // empty-model state after the user picks a provider. useEffect(() => { if (!value.provider) return + if (gatewayDirectModel) { + if (value.model) onChange({ ...value, model: null }) + return + } if (!models.length) return if (value.model && models.includes(value.model)) return onChange({ ...value, model: models[0] }) // eslint-disable-next-line react-hooks/exhaustive-deps - }, [value.provider, models]) + }, [value.provider, models, gatewayDirectModel]) // When the audio-capable toggle flips ON after the user has already // picked a non-audio provider/model (e.g. Anthropic + Claude), clear @@ -310,44 +324,63 @@ export default function ProviderModelPicker({ )}
- - - {audioCapableOnly && - kind === 'llm' && - value.provider && - rawModels.length > 0 && - models.length === 0 && ( -

- This provider has no audio-capable Chat Completions - models. Use OpenAI's gpt-4o-audio-preview / - gpt-4o-mini-audio-preview, or a Google Gemini 1.5+ model. + {gatewayDirectModel ? ( + <> + +

+ {gatewayDirectModel} +
+

+ Model is fixed on the integration — Bifrost gateway routing applies.

- )} + + ) : ( + <> + + + {audioCapableOnly && + kind === 'llm' && + value.provider && + rawModels.length > 0 && + models.length === 0 && ( +

+ This provider has no audio-capable Chat Completions + models. Use OpenAI's gpt-4o-audio-preview / + gpt-4o-mini-audio-preview, or a Google Gemini 1.5+ model. +

+ )} + + )}
{showCredentialPicker && ( @@ -377,7 +410,7 @@ export default function ProviderModelPicker({ )} - {kind === 'llm' && showAdvancedOptions && value.provider && ( + {kind === 'llm' && showAdvancedOptions && value.provider && !gatewayDirectModel && ( { const response = await this.client.post( `/api/v1/prompt-partials/${partialId}/flowchart`, { provider: options?.provider, model: options?.model, + credential_id: options?.credential_id, regenerate: options?.regenerate ?? false, }, ) @@ -3774,13 +3786,14 @@ class ApiClient { async mapAgentFlowchartPromptSections( partialId: string, - options?: { provider?: string; model?: string }, + options?: { provider?: string; model?: string; credential_id?: string }, ): Promise { const response = await this.client.post( `/api/v1/prompt-partials/${partialId}/flowchart/prompt-map`, { provider: options?.provider, model: options?.model, + credential_id: options?.credential_id, }, ) return response.data @@ -3858,6 +3871,7 @@ class ApiClient { format_style?: string provider?: string model?: string + credential_id?: string llm_config?: LLMGenerationConfig | null }): Promise<{ content: string; provider: string; model: string }> { const response = await this.client.post('/api/v1/prompt-partials/generate', data) @@ -3869,6 +3883,7 @@ class ApiClient { instructions?: string provider?: string model?: string + credential_id?: string llm_config?: LLMGenerationConfig | null }): Promise<{ content: string; provider: string; model: string }> { const response = await this.client.post('/api/v1/prompt-partials/improve', data) diff --git a/frontend/src/lib/gatewayRouting.ts b/frontend/src/lib/gatewayRouting.ts new file mode 100644 index 0000000..0f85db7 --- /dev/null +++ b/frontend/src/lib/gatewayRouting.ts @@ -0,0 +1,43 @@ +import type { AIProvider } from '../types/api' + +type GatewayCredential = Pick< + AIProvider, + 'id' | 'provider' | 'gateway_model' | 'routing_mode' | 'effective_routing' +> + +export function routesViaGateway( + credential?: GatewayCredential | null, +): boolean { + if (!credential) return false + const effective = credential.effective_routing + if ( + effective === 'bifrost' || + effective === 'litellm_proxy' || + effective === 'gateway' + ) { + return true + } + return credential.routing_mode === 'gateway' +} + +export function usesGatewayDirectModel( + credential?: GatewayCredential | null, +): boolean { + const gatewayModel = credential?.gateway_model?.trim() + if (!gatewayModel) return false + return routesViaGateway(credential) +} + +export function resolveActiveAIProvider( + aiProviders: AIProvider[], + providerKey: string, + credentialId?: string | null, +): AIProvider | undefined { + const rows = aiProviders.filter( + (p) => p.is_active && p.provider.toLowerCase() === providerKey.toLowerCase(), + ) + if (credentialId) { + return rows.find((p) => p.id === credentialId) + } + return rows.find((p) => p.is_default) ?? rows[0] +} diff --git a/frontend/src/pages/configurations/Integrations.tsx b/frontend/src/pages/configurations/Integrations.tsx index 0e8369a..d2e4e0b 100644 --- a/frontend/src/pages/configurations/Integrations.tsx +++ b/frontend/src/pages/configurations/Integrations.tsx @@ -4,7 +4,7 @@ import type { TelephonyIntegrationResponse } from '../../lib/api' import { useState, useEffect, useRef, type ReactNode } from 'react' import { createPortal } from 'react-dom' import { Plus, Trash2, X, AlertCircle, Plug, Edit, Brain, ChevronDown, Phone, Star, Network } from 'lucide-react' -import { IntegrationCreate, IntegrationPlatform, Integration, AIProvider, AIProviderCreate, ModelProvider, TelephonyProvider } from '../../types/api' +import { IntegrationCreate, IntegrationPlatform, Integration, AIProvider, AIProviderCreate, AIProviderUpdate, ModelProvider, TelephonyProvider, CredentialRoutingMode, GatewayInterfaceMode } from '../../types/api' import type { LLMGatewayMode, LLMGatewaySettings, @@ -55,6 +55,15 @@ export default function Integrations() { const [apiKey, setApiKey] = useState('') const [publicKey, setPublicKey] = useState('') const [name, setName] = useState('') + const [credentialRoutingMode, setCredentialRoutingMode] = useState('inherit') + const [gatewayModel, setGatewayModel] = useState('') + const [gatewayInterface, setGatewayInterface] = useState('inherit') + const [gatewayBaseUrl, setGatewayBaseUrl] = useState('') + const [gatewayAuthHeader, setGatewayAuthHeader] = useState('') + const [gatewayAuthSecretEnv, setGatewayAuthSecretEnv] = useState('') + const [gatewayAuthSecret, setGatewayAuthSecret] = useState('') + const [clearGatewayAuthSecret, setClearGatewayAuthSecret] = useState(false) + const [gatewayExtraHeadersJson, setGatewayExtraHeadersJson] = useState('') const [showDeleteModal, setShowDeleteModal] = useState(false) const [showDeleteAIProviderModal, setShowDeleteAIProviderModal] = useState(false) const [showDeleteTelephonyModal, setShowDeleteTelephonyModal] = useState(false) @@ -76,6 +85,7 @@ export default function Integrations() { const [llmGatewayMode, setLlmGatewayMode] = useState('inherit') const [llmGatewayType, setLlmGatewayType] = useState('inherit') + const [llmGatewayInterface, setLlmGatewayInterface] = useState('inherit') const [llmGatewayBaseUrl, setLlmGatewayBaseUrl] = useState('') const [llmGatewayVirtualKey, setLlmGatewayVirtualKey] = useState('') const [llmGatewayMasterKey, setLlmGatewayMasterKey] = useState('') @@ -87,6 +97,7 @@ export default function Integrations() { if (!llmGatewaySettings) return setLlmGatewayMode(llmGatewaySettings.mode) setLlmGatewayType(llmGatewaySettings.gateway_type) + setLlmGatewayInterface(llmGatewaySettings.gateway_interface || 'inherit') setLlmGatewayBaseUrl(llmGatewaySettings.base_url || '') setLlmGatewayVirtualKey('') setLlmGatewayMasterKey('') @@ -104,12 +115,50 @@ export default function Integrations() { syncLlmGatewayFormFromSettings() } + const llmGatewayInterfaceLabel = (iface?: string) => { + if (iface === 'native_openai') return 'Native OpenAI' + if (iface === 'litellm_shim') return 'LiteLLM shim' + return 'Inherit' + } + const llmGatewayRoutingLabel = (routing: LLMGatewaySettings['effective_routing']) => { if (routing === 'bifrost') return 'Bifrost' if (routing === 'litellm_proxy') return 'LiteLLM Proxy' return 'Direct' } + const credentialRoutingLabel = (routing?: string) => { + if (routing === 'bifrost') return 'Bifrost' + if (routing === 'litellm_proxy') return 'LiteLLM Proxy' + if (routing === 'gateway') return 'Gateway' + if (routing === 'direct') return 'Direct' + return 'Inherit' + } + + const formatGatewayExtraHeadersJson = (headers?: Record | null) => { + if (!headers || Object.keys(headers).length === 0) return '' + return JSON.stringify(headers, null, 2) + } + + const parseGatewayExtraHeadersJson = ( + json: string, + ): Record | null => { + const trimmed = json.trim() + if (!trimmed) return null + const parsed = JSON.parse(trimmed) as unknown + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('Gateway extra headers must be a JSON object') + } + const result: Record = {} + for (const [key, value] of Object.entries(parsed)) { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`Header "${key}" must have a non-empty string value`) + } + result[key] = value.trim() + } + return Object.keys(result).length > 0 ? result : null + } + const renderModal = (content: ReactNode) => { if (typeof document === 'undefined') return null return createPortal(content, document.body) @@ -153,6 +202,7 @@ export default function Integrations() { apiClient.updateLLMGatewaySettings({ mode: llmGatewayMode, gateway_type: llmGatewayType, + gateway_interface: llmGatewayInterface, base_url: llmGatewayBaseUrl.trim() || null, virtual_key: llmGatewayVirtualKey.trim() || undefined, master_key: llmGatewayMasterKey.trim() || undefined, @@ -223,7 +273,7 @@ export default function Integrations() { }) const updateAIProviderMutation = useMutation({ - mutationFn: ({ id, data }: { id: string; data: Partial }) => apiClient.updateAIProvider(id, data), + mutationFn: ({ id, data }: { id: string; data: Partial }) => apiClient.updateAIProvider(id, data), onSuccess: () => { queryClient.invalidateQueries({ queryKey: ['aiproviders'] }); showToast('AI Provider updated successfully!', 'success'); resetForm() }, onError: (error: any) => { showToast(`Failed to update provider: ${error.response?.data?.detail || error.message}`, 'error') }, }) @@ -309,6 +359,9 @@ export default function Integrations() { setShowModal(false); setIsEditMode(false); setIntegrationType(null); setSelectedIntegration(null); setSelectedAIProvider(null) setSelectedPlatform(null); setSelectedProvider(null); setShowProviderDropdown(false); setShowPlatformDropdown(false) setApiKey(''); setPublicKey(''); setName('') + setCredentialRoutingMode('inherit'); setGatewayModel(''); setGatewayInterface('inherit'); setGatewayBaseUrl('') + setGatewayAuthHeader(''); setGatewayAuthSecretEnv(''); setGatewayAuthSecret(''); setClearGatewayAuthSecret(false) + setGatewayExtraHeadersJson('') setSelectedTelephonyProvider(null); setTelephonyAuthId(''); setTelephonyAuthToken(''); setTelephonyVerifyAppUuid(''); setTelephonyVoiceAppId(''); setTelephonySipDomain('') setEditingTelephonyConfigId(null); setTelephonyName('') } @@ -320,13 +373,19 @@ export default function Integrations() { setName(integration.name || '') setApiKey('') // Don't pre-fill API key for security setPublicKey(integration.public_key || '') + setCredentialRoutingMode(integration.routing_mode || 'inherit') setIsEditMode(true) setShowModal(true) } const handleEditAIProvider = (provider: AIProvider) => { setIntegrationType('ai_provider'); setSelectedAIProvider(provider); setSelectedProvider(provider.provider) - setName(provider.name || ''); setApiKey(''); setShowProviderDropdown(false); setIsEditMode(true); setShowModal(true) + setName(provider.name || ''); setApiKey(''); setCredentialRoutingMode(provider.routing_mode || 'inherit') + setGatewayModel(provider.gateway_model || ''); setGatewayInterface(provider.gateway_interface || 'inherit') + setGatewayBaseUrl(provider.gateway_base_url || ''); setGatewayAuthHeader(provider.gateway_auth_header || '') + setGatewayAuthSecretEnv(provider.gateway_auth_secret_env || ''); setGatewayAuthSecret(''); setClearGatewayAuthSecret(false) + setGatewayExtraHeadersJson(formatGatewayExtraHeadersJson(provider.gateway_extra_headers)) + setShowProviderDropdown(false); setIsEditMode(true); setShowModal(true) } const handleEditTelephony = (config?: TelephonyIntegrationResponse) => { @@ -349,17 +408,67 @@ export default function Integrations() { if (name !== (selectedIntegration.name || '')) updateData.name = name || undefined if (apiKey) updateData.api_key = apiKey if (publicKey !== (selectedIntegration.public_key || '')) updateData.public_key = publicKey || undefined + if (credentialRoutingMode !== (selectedIntegration.routing_mode || 'inherit')) { + updateData.routing_mode = credentialRoutingMode + } if (Object.keys(updateData).length > 0) updateIntegrationMutation.mutate({ id: selectedIntegration.id, data: updateData }) else resetForm() } else { if (!selectedPlatform || !apiKey) return - createIntegrationMutation.mutate({ platform: selectedPlatform as IntegrationPlatform, api_key: apiKey, public_key: publicKey || undefined, name: name || undefined }) + createIntegrationMutation.mutate({ + platform: selectedPlatform as IntegrationPlatform, + api_key: apiKey, + public_key: publicKey || undefined, + name: name || undefined, + routing_mode: credentialRoutingMode, + }) } } else if (integrationType === 'ai_provider') { + let parsedGatewayExtraHeaders: Record | null = null + try { + parsedGatewayExtraHeaders = parseGatewayExtraHeadersJson(gatewayExtraHeadersJson) + } catch (error: any) { + showToast(error?.message || 'Invalid gateway extra headers JSON', 'error') + return + } + if (isEditMode && selectedAIProvider) { - const updateData: Partial = {} + const updateData: Partial = {} if (apiKey.trim()) updateData.api_key = apiKey if (name !== (selectedAIProvider.name || '')) updateData.name = name || null + if (credentialRoutingMode !== (selectedAIProvider.routing_mode || 'inherit')) { + updateData.routing_mode = credentialRoutingMode + } + const trimmedGatewayModel = gatewayModel.trim() + if (trimmedGatewayModel !== (selectedAIProvider.gateway_model || '')) { + updateData.gateway_model = trimmedGatewayModel || null + } + if (gatewayInterface !== (selectedAIProvider.gateway_interface || 'inherit')) { + updateData.gateway_interface = gatewayInterface + } + const trimmedGatewayBaseUrl = gatewayBaseUrl.trim() + if (trimmedGatewayBaseUrl !== (selectedAIProvider.gateway_base_url || '')) { + updateData.gateway_base_url = trimmedGatewayBaseUrl || null + } + const trimmedGatewayAuthHeader = gatewayAuthHeader.trim() + if (trimmedGatewayAuthHeader !== (selectedAIProvider.gateway_auth_header || '')) { + updateData.gateway_auth_header = trimmedGatewayAuthHeader || null + } + const trimmedGatewayAuthSecretEnv = gatewayAuthSecretEnv.trim() + if (trimmedGatewayAuthSecretEnv !== (selectedAIProvider.gateway_auth_secret_env || '')) { + updateData.gateway_auth_secret_env = trimmedGatewayAuthSecretEnv || null + } + if (clearGatewayAuthSecret) { + updateData.clear_gateway_auth_secret = true + } else if (gatewayAuthSecret.trim()) { + updateData.gateway_auth_secret = gatewayAuthSecret.trim() + } + const existingExtraHeadersJson = formatGatewayExtraHeadersJson( + selectedAIProvider.gateway_extra_headers, + ) + if (gatewayExtraHeadersJson.trim() !== existingExtraHeadersJson.trim()) { + updateData.gateway_extra_headers = parsedGatewayExtraHeaders + } if (Object.keys(updateData).length === 0) { resetForm(); return } updateAIProviderMutation.mutate({ id: selectedAIProvider.id, data: updateData }) } else { @@ -367,7 +476,7 @@ export default function Integrations() { showToast('Please select a provider', 'error') return } - if (!aiIntegrationGatewayManaged && !apiKey.trim()) { + if (aiProviderRequiresApiKey && !apiKey.trim()) { showToast('Please enter an API key', 'error') return } @@ -375,6 +484,14 @@ export default function Integrations() { provider: selectedProvider, api_key: apiKey.trim() || undefined, name: name || null, + routing_mode: credentialRoutingMode, + gateway_model: gatewayModel.trim() || undefined, + gateway_interface: gatewayInterface, + gateway_base_url: gatewayBaseUrl.trim() || undefined, + gateway_auth_header: gatewayAuthHeader.trim() || undefined, + gateway_auth_secret_env: gatewayAuthSecretEnv.trim() || undefined, + gateway_auth_secret: gatewayAuthSecret.trim() || undefined, + gateway_extra_headers: parsedGatewayExtraHeaders || undefined, }) } } else if (integrationType === 'telephony_provider') { @@ -462,9 +579,14 @@ export default function Integrations() { aiIntegrationProviders.length > 0 || hasTelephony - const aiIntegrationGatewayManaged = Boolean( - llmGatewaySettings?.gateway_managed_credentials, - ) + const showGatewayModelField = + integrationType === 'ai_provider' && + (credentialRoutingMode === 'gateway' || + (credentialRoutingMode === 'inherit' && + llmGatewaySettings?.effective_routing && + llmGatewaySettings.effective_routing !== 'direct')) + + const aiProviderRequiresApiKey = credentialRoutingMode === 'direct' const getPlatformInfo = (platformId: IntegrationPlatform) => { return platforms.find(p => p.id === platformId) @@ -552,6 +674,16 @@ export default function Integrations() { Default )} + {integration.routing_mode && integration.routing_mode !== 'inherit' && ( + + {integration.routing_mode === 'gateway' ? 'Gateway' : 'Direct'} + + )} + {integration.effective_routing && integration.effective_routing !== 'direct' && ( + + {credentialRoutingLabel(integration.effective_routing)} + + )} {!integration.is_active && Inactive} @@ -630,6 +762,26 @@ export default function Integrations() { Gateway managed )} + {provider.routing_mode && provider.routing_mode !== 'inherit' && ( + + {provider.routing_mode === 'gateway' ? 'Gateway' : 'Direct'} + + )} + {provider.effective_routing && provider.effective_routing !== 'direct' && ( + + {credentialRoutingLabel(provider.effective_routing)} + + )} + {provider.effective_gateway_interface === 'native_openai' && ( + + Native API + + )} + {provider.gateway_model && ( + + {provider.gateway_model} + + )} {!provider.is_active && Inactive} @@ -811,6 +963,26 @@ export default function Integrations() { + {effectiveLlmGatewayType === 'bifrost' && ( +
+ + +

+ Use native OpenAI-compatible for custom Bifrost models (e.g. Gemma) that do not work through the /litellm shim. +

+
+ )} +

{effectiveLlmGatewayType === 'litellm_proxy' ? 'LiteLLM Proxy base URL (e.g. http://localhost:4000). Leave blank to inherit the platform URL.' - : 'Bifrost URL must include the /litellm path. Leave blank to inherit the platform URL.'} + : (llmGatewayInterface === 'native_openai' || llmGatewaySettings?.effective_gateway_interface === 'native_openai') + ? 'Bifrost host root only (e.g. http://localhost:8080). /v1 is added automatically. Do not include /v1/chat/completions.' + : 'Bifrost URL with /litellm path. Leave blank to inherit the platform URL.'}

@@ -1006,6 +1182,21 @@ export default function Integrations() { setName(e.target.value)} className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500" placeholder="Integration name" /> +
+ + +

+ Applies to batch LLM workloads when this credential is used. Real-time voice agents always use direct API keys. +

+
setApiKey(e.target.value)} className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500" @@ -1053,35 +1244,163 @@ export default function Integrations() { setName(e.target.value)} className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500" placeholder="e.g., OpenAI Production Key" />
+
+ + +

+ Controls whether batch/eval LLM calls use your org gateway or call the provider directly. +

+
+ {showGatewayModelField && ( + <> +
+ + setGatewayModel(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500" + placeholder="e.g., production-gpt4 or openai/gpt-4o" + /> +

+ Bifrost custom model ID sent when routing via gateway. Leave blank to use the workload-selected model. +

+
+
+ + +

+ Override how this credential reaches Bifrost. Use native for custom models that fail through /litellm. +

+
+
+ + setGatewayBaseUrl(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500" + placeholder="e.g. http://localhost:8080" + /> + {gatewayInterface === 'native_openai' && ( +

+ Host root only. /v1 is added automatically — do not include /v1/chat/completions. +

+ )} +
+
+ + setGatewayAuthHeader(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500" + placeholder="x-bf-vk" + /> +

+ Header name for Bifrost auth. Defaults to x-bf-vk when blank. +

+
+
+ + setGatewayAuthSecretEnv(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500" + placeholder="BIFROST_VIRTUAL_KEY" + /> +

+ Read the auth secret from this environment variable at runtime (e.g. K8s secret). Overrides org virtual key. +

+
+
+ + setGatewayAuthSecret(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500" + placeholder="Alternative to env var — encrypted at rest" + /> + {isEditMode && selectedAIProvider?.has_gateway_auth_secret && ( + + )} +
+
+ +