feat: updating gateway integration

app/api/v1/api.py

@@ -42,6 +42,7 @@
     workspaces,
     workspace_iam,
     dashboard,
+    llm_gateway,
 )
 
 api_router = APIRouter()
@@ -87,3 +88,4 @@
 api_router.include_router(workspaces.router)
 api_router.include_router(workspace_iam.router)
 api_router.include_router(dashboard.router)
+api_router.include_router(llm_gateway.router)

app/api/v1/routes/aiproviders.py

@@ -12,7 +12,7 @@
 from fastapi import APIRouter, Depends, HTTPException, status
 from sqlalchemy import desc, func
 from sqlalchemy.orm import Session
-from typing import List
+from typing import List, Optional
 from uuid import UUID
 
 from app.dependencies import get_db, get_organization_id
@@ -22,21 +22,44 @@
 )
 from app.core.encryption import encrypt_api_key
 from app.services.credentials.resolver import clear_other_defaults
+from app.services.ai.llm_gateway import (
+    GATEWAY_MANAGED_KEY_SENTINEL,
+    gateway_managed_credentials_enabled,
+    is_gateway_managed_stored_key,
+)
 
 router = APIRouter(prefix="/aiproviders", tags=["aiproviders"])
 
 
-def _scrub_for_response(db: Session, instance: AIProvider) -> AIProvider:
+def _scrub_for_response(db: Session, instance: AIProvider) -> AIProviderResponse:
     """Detach the row from the session before clearing ``api_key``.
 
     The same SQLAlchemy session is reused across requests in tests; if we
     leave the instance attached and assign ``api_key = None``, SQLAlchemy
     treats it as a pending UPDATE and tries to flush ``api_key=NULL`` on
     the next request — which violates the column's NOT NULL constraint.
     """
+    gateway_managed = is_gateway_managed_stored_key(instance.api_key)
     db.expunge(instance)
     instance.api_key = None
-    return instance
+    response = AIProviderResponse.model_validate(instance)
+    return response.model_copy(update={"gateway_managed": gateway_managed})
+
+
+def _encrypt_provider_api_key(api_key: Optional[str]) -> str:
+    trimmed = (api_key or "").strip()
+    if trimmed:
+        return encrypt_api_key(trimmed)
+    if gateway_managed_credentials_enabled():
+        return encrypt_api_key(GATEWAY_MANAGED_KEY_SENTINEL)
+    raise HTTPException(
+        status_code=status.HTTP_400_BAD_REQUEST,
+        detail=(
+            "api_key is required. When LLM gateway-managed credentials "
+            "are enabled (passthrough_provider_keys: false), you can omit "
+            "the key and provider secrets will be resolved by the gateway."
+        ),
+    )
 
 
 @router.post("", response_model=AIProviderResponse, status_code=status.HTTP_201_CREATED, operation_id="createAIProvider")
@@ -63,7 +86,7 @@ async def create_aiprovider(
     requested_default = bool(aiprovider.is_default)
     will_be_default = requested_default or existing_default is None
 
-    encrypted_api_key = encrypt_api_key(aiprovider.api_key)
+    encrypted_api_key = _encrypt_provider_api_key(aiprovider.api_key)
     db_aiprovider = AIProvider(
         organization_id=organization_id,
         provider=provider_value,

app/api/v1/routes/llm_gateway.py

@@ -0,0 +1,94 @@
+"""Organization-level LLM gateway settings."""
+
+from typing import Literal, Optional
+from uuid import UUID
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy.orm import Session
+
+from app.dependencies import get_db, get_organization_id
+from app.services.ai.llm_gateway_settings import (
+    GatewayMode,
+    GatewayTypeOverride,
+    get_org_settings,
+    set_org_settings,
+)
+
+router = APIRouter(prefix="/organizations/llm-gateway", tags=["LLM Gateway"])
+
+
+class LLMGatewaySettingsResponse(BaseModel):
+    mode: GatewayMode
+    gateway_type: GatewayTypeOverride
+    base_url: Optional[str] = None
+    has_virtual_key: bool = False
+    has_master_key: bool = False
+    platform_enabled: bool = False
+    platform_gateway_type: Literal["bifrost", "litellm_proxy"] = "bifrost"
+    platform_base_url: Optional[str] = None
+    effective_routing: Literal["direct", "bifrost", "litellm_proxy"]
+    effective_gateway_type: Optional[Literal["bifrost", "litellm_proxy"]] = None
+    effective_base_url: Optional[str] = None
+    effective_has_virtual_key: bool = False
+    effective_has_master_key: bool = False
+    gateway_managed_credentials: bool = False
+
+
+class LLMGatewaySettingsUpdate(BaseModel):
+    mode: GatewayMode = Field(
+        ...,
+        description="inherit: use platform default; enabled: force gateway; disabled: opt out",
+    )
+    gateway_type: GatewayTypeOverride = Field(
+        default="inherit",
+        description="inherit: use platform gateway type; bifrost or litellm_proxy to override",
+    )
+    base_url: Optional[str] = Field(
+        default=None,
+        description="Optional org-specific gateway URL override.",
+    )
+    virtual_key: Optional[str] = Field(
+        default=None,
+        description="Optional Bifrost virtual key (x-bf-vk). Omit to keep existing.",
+    )
+    master_key: Optional[str] = Field(
+        default=None,
+        description="Optional LiteLLM Proxy master key. Omit to keep existing.",
+    )
+    clear_virtual_key: bool = Field(
+        default=False,
+        description="When true, remove the stored org virtual key.",
+    )
+    clear_master_key: bool = Field(
+        default=False,
+        description="When true, remove the stored org master key.",
+    )
+
+
+@router.get("", response_model=LLMGatewaySettingsResponse)
+def get_llm_gateway_settings(
+    organization_id: UUID = Depends(get_organization_id),
+    db: Session = Depends(get_db),
+):
+    return LLMGatewaySettingsResponse(**get_org_settings(organization_id, db))
+
+
+@router.put("", response_model=LLMGatewaySettingsResponse)
+def update_llm_gateway_settings(
+    body: LLMGatewaySettingsUpdate,
+    organization_id: UUID = Depends(get_organization_id),
+    db: Session = Depends(get_db),
+):
+    result = set_org_settings(
+        organization_id,
+        db,
+        mode=body.mode,
+        gateway_type=body.gateway_type,
+        base_url=body.base_url,
+        virtual_key=body.virtual_key,
+        master_key=body.master_key,
+        clear_virtual_key=body.clear_virtual_key,
+        clear_master_key=body.clear_master_key,
+    )
+    return LLMGatewaySettingsResponse(**result)

app/config.py

@@ -163,6 +163,14 @@ class Settings(BaseSettings):
     JUDGE_ALIGNMENT_ENABLED: bool = True
     JUDGE_ALIGNMENT_CSV_MAX_ROWS: int = 5000
 
+    # LLM gateway (optional platform-wide proxy for batch LLM calls).
+    LLM_GATEWAY_ENABLED: bool = False
+    LLM_GATEWAY_TYPE: str = "bifrost"  # bifrost | litellm_proxy
+    LLM_GATEWAY_BASE_URL: Optional[str] = None
+    LLM_GATEWAY_VIRTUAL_KEY: Optional[str] = None
+    LLM_GATEWAY_MASTER_KEY: Optional[str] = None
+    LLM_GATEWAY_PASSTHROUGH_PROVIDER_KEYS: bool = True
+
     model_config = SettingsConfigDict(
         env_file=".env",
         env_file_encoding="utf-8",
@@ -600,6 +608,28 @@ def load_config_from_file(config_path: str) -> None:
         if "csv_max_rows" in ja_cfg:
             settings.JUDGE_ALIGNMENT_CSV_MAX_ROWS = int(ja_cfg["csv_max_rows"])
 
+    def _apply_llm_gateway_settings(gateway_cfg: dict, *, gateway_type: str) -> None:
+        if "enabled" in gateway_cfg:
+            settings.LLM_GATEWAY_ENABLED = bool(gateway_cfg["enabled"])
+        settings.LLM_GATEWAY_TYPE = gateway_type
+        if gateway_cfg.get("base_url"):
+            settings.LLM_GATEWAY_BASE_URL = gateway_cfg["base_url"]
+        if gateway_cfg.get("virtual_key"):
+            settings.LLM_GATEWAY_VIRTUAL_KEY = gateway_cfg["virtual_key"]
+        if gateway_cfg.get("master_key"):
+            settings.LLM_GATEWAY_MASTER_KEY = gateway_cfg["master_key"]
+        if "passthrough_provider_keys" in gateway_cfg:
+            settings.LLM_GATEWAY_PASSTHROUGH_PROVIDER_KEYS = bool(
+                gateway_cfg["passthrough_provider_keys"]
+            )
+
+    if "llm_gateway" in config_data:
+        llm_cfg = config_data["llm_gateway"]
+        gateway_type = (llm_cfg.get("type") or "bifrost").strip().lower()
+        if gateway_type not in ("bifrost", "litellm_proxy"):
+            gateway_type = "bifrost"
+        _apply_llm_gateway_settings(llm_cfg, gateway_type=gateway_type)
+
     if "operational" in config_data:
         operational_config = config_data["operational"]
         if "public" in operational_config:

app/migrations/050_llm_gateway_settings.py

@@ -0,0 +1,48 @@
+"""
+Migration: Add per-organization LLM gateway settings JSON column.
+"""
+
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+description = "Add organizations.llm_gateway_settings JSON for per-org LLM gateway overrides"
+
+
+def _column_exists(db: Session, column_name: str) -> bool:
+    row = db.execute(
+        text(
+            """
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_name = 'organizations'
+              AND column_name = :column_name
+            """
+        ),
+        {"column_name": column_name},
+    ).first()
+    return row is not None
+
+
+def upgrade(db: Session):
+    if _column_exists(db, "llm_gateway_settings"):
+        print("Column llm_gateway_settings already exists on organizations, skipping...")
+        return
+
+    db.execute(
+        text(
+            """
+        ALTER TABLE organizations
+        ADD COLUMN llm_gateway_settings JSON
+        """
+        )
+    )
+
+    db.commit()
+    print("Added llm_gateway_settings column to organizations")
+
+
+def downgrade(db: Session):
+    db.execute(
+        text("ALTER TABLE organizations DROP COLUMN IF EXISTS llm_gateway_settings")
+    )
+    db.commit()

app/migrations/051_rename_bifrost_gateway_settings_column.py

@@ -0,0 +1,72 @@
+"""
+Migration: Rename legacy organizations.bifrost_gateway_settings column.
+
+For databases that applied an earlier 050 migration before the column was
+renamed to llm_gateway_settings.
+"""
+
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+description = (
+    "Rename organizations.bifrost_gateway_settings to llm_gateway_settings"
+)
+
+
+def _column_exists(db: Session, column_name: str) -> bool:
+    row = db.execute(
+        text(
+            """
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_name = 'organizations'
+              AND column_name = :column_name
+            """
+        ),
+        {"column_name": column_name},
+    ).first()
+    return row is not None
+
+
+def upgrade(db: Session):
+    if _column_exists(db, "llm_gateway_settings"):
+        print("Column llm_gateway_settings already exists, skipping rename...")
+        return
+
+    if not _column_exists(db, "bifrost_gateway_settings"):
+        print("Legacy bifrost_gateway_settings column not found, skipping rename...")
+        return
+
+    db.execute(
+        text(
+            """
+        ALTER TABLE organizations
+        RENAME COLUMN bifrost_gateway_settings TO llm_gateway_settings
+        """
+        )
+    )
+    db.commit()
+    print(
+        "Renamed organizations.bifrost_gateway_settings "
+        "to llm_gateway_settings"
+    )
+
+
+def downgrade(db: Session):
+    if _column_exists(db, "bifrost_gateway_settings"):
+        print("Column bifrost_gateway_settings already exists, skipping rename...")
+        return
+
+    if not _column_exists(db, "llm_gateway_settings"):
+        print("Column llm_gateway_settings not found, skipping rename...")
+        return
+
+    db.execute(
+        text(
+            """
+        ALTER TABLE organizations
+        RENAME COLUMN llm_gateway_settings TO bifrost_gateway_settings
+        """
+        )
+    )
+    db.commit()

app/models/database.py

@@ -54,6 +54,8 @@ class Organization(Base):
     # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int}
     # Falls back to system defaults (20 / 50) when null.
     judge_alignment_settings = Column(JSON, nullable=True)
+    # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys).
+    llm_gateway_settings = Column(JSON, nullable=True)
     created_at = Column(DateTime(timezone=True), server_default=func.now())
     updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
 

app/models/schemas.py

@@ -669,7 +669,13 @@ class S3UploadResponse(BaseModel):
 class AIProviderCreate(BaseModel):
     """Schema for creating an AI Provider."""
     provider: ModelProvider
-    api_key: str = Field(..., min_length=1)
+    api_key: Optional[str] = Field(
+        None,
+        description=(
+            "Provider API key. Optional when the platform uses Bifrost "
+            "gateway-managed credentials (passthrough_provider_keys: false)."
+        ),
+    )
     name: Optional[str] = None
     is_default: Optional[bool] = Field(
         None,
@@ -679,6 +685,14 @@ class AIProviderCreate(BaseModel):
         ),
     )
 
+    @field_validator("api_key")
+    @classmethod
+    def validate_api_key(cls, v: Optional[str]) -> Optional[str]:
+        if v is None:
+            return v
+        trimmed = v.strip()
+        return trimmed or None
+
 
 class AIProviderUpdate(BaseModel):
     """Schema for updating an AI Provider."""
@@ -695,6 +709,7 @@ class AIProviderResponse(BaseModel):
     name: Optional[str]
     is_active: bool
     is_default: bool = False
+    gateway_managed: bool = False
     created_at: datetime
     updated_at: datetime
     last_tested_at: Optional[datetime]