diff --git a/AGENTS.md b/AGENTS.md index 93c2c13..edacd93 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,11 +52,15 @@ cherry-evals/ │ └── ARCHITECTURE.md # Technical architecture and ADRs │ ├── api/ # FastAPI REST API layer -│ ├── main.py # App factory and router registration +│ ├── main.py # App factory, CORS, router registration +│ ├── deps.py # Auth, rate limits, quota enforcement │ ├── routes/ # Endpoint definitions -│ │ └── analytics.py # Analytics endpoints -│ ├── models/ # Pydantic request/response schemas -│ └── deps.py # Dependency injection +│ │ ├── account.py # User profile + usage stats +│ │ ├── analytics.py # Analytics endpoints +│ │ ├── api_keys.py # API key CRUD +│ │ ├── billing.py # Polar.sh webhook +│ │ └── ... # collections, search, export, agents, etc. +│ └── models/ # Pydantic request/response schemas │ ├── cherry_evals/ # Core application package │ ├── config.py # Pydantic settings @@ -424,6 +428,21 @@ LANGFUSE_PUBLIC_KEY=your-public-key LANGFUSE_SECRET_KEY=your-secret-key LANGFUSE_BASE_URL=https://cloud.langfuse.com +# Auth (Supabase) +SUPABASE_URL=https://your-project.supabase.co +SUPABASE_JWT_SECRET=your-jwt-secret + +# Billing (Polar.sh) +POLAR_WEBHOOK_SECRET=your-polar-webhook-secret +POLAR_PRO_PRODUCT_ID=your-polar-pro-product-id +POLAR_ULTRA_PRODUCT_ID=your-polar-ultra-product-id + +# Auth toggle (default True — set False for local dev) +AUTH_ENABLED=True + +# CORS origins (comma-separated) +CORS_ORIGINS=https://app.cherryevals.com,http://localhost:5173 + # Optional CHERRY_DATA_DIR=./data CHERRY_LOG_LEVEL=INFO diff --git a/agents/export_agent.py b/agents/export_agent.py index 33b2d0a..6ac7c0e 100644 --- a/agents/export_agent.py +++ b/agents/export_agent.py @@ -117,6 +117,7 @@ def _compile_convert_function(code: str) -> callable | None: """ import csv import io + import re safe_builtins = { "True": True, @@ -156,6 +157,7 @@ def _compile_convert_function(code: str) -> callable | None: "json": json, "csv": csv, "io": io, + "re": re, } namespace: dict = {} exec(code, safe_globals, namespace) # noqa: S102 diff --git a/agents/ingestion_agent.py b/agents/ingestion_agent.py index 5833167..f8441c3 100644 --- a/agents/ingestion_agent.py +++ b/agents/ingestion_agent.py @@ -134,8 +134,10 @@ def _compile_parse_function(code: str) -> callable | None: } try: + import re + namespace: dict = {} - exec(code, {"__builtins__": safe_builtins}, namespace) # noqa: S102 + exec(code, {"__builtins__": safe_builtins, "re": re}, namespace) # noqa: S102 fn = namespace.get("parse_row") if fn is None or not callable(fn): logger.warning("Generated code does not define a callable 'parse_row'") diff --git a/alembic/versions/004_add_users_and_api_keys.py b/alembic/versions/004_add_users_and_api_keys.py new file mode 100644 index 0000000..3786036 --- /dev/null +++ b/alembic/versions/004_add_users_and_api_keys.py @@ -0,0 +1,89 @@ +"""add users and api_keys tables + +Revision ID: 004_add_users_and_api_keys +Revises: 003_add_curation_events +Create Date: 2026-03-10 00:00:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "004_add_users_and_api_keys" +down_revision: str | Sequence[str] | None = "003_add_curation_events" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "users", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("supabase_id", sa.String(length=255), nullable=False), + sa.Column("email", sa.String(length=255), nullable=False), + sa.Column("tier", sa.String(length=50), nullable=False, server_default="free"), + sa.Column("polar_customer_id", sa.String(length=255), nullable=True), + sa.Column("polar_subscription_id", sa.String(length=255), nullable=True), + sa.Column("subscription_status", sa.String(length=50), nullable=True), + sa.Column("llm_calls_today", sa.Integer(), nullable=False, server_default="0"), + sa.Column("semantic_searches_today", sa.Integer(), nullable=False, server_default="0"), + sa.Column( + "quota_reset_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_users_id"), "users", ["id"], unique=False) + op.create_index(op.f("ix_users_supabase_id"), "users", ["supabase_id"], unique=True) + + op.create_table( + "api_keys", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("key_prefix", sa.String(length=20), nullable=False), + sa.Column("key_hash", sa.String(length=64), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False, server_default="Default"), + sa.Column("last_used_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.text("true")), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_api_keys_id"), "api_keys", ["id"], unique=False) + op.create_index(op.f("ix_api_keys_user_id"), "api_keys", ["user_id"], unique=False) + op.create_index(op.f("ix_api_keys_key_hash"), "api_keys", ["key_hash"], unique=True) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index(op.f("ix_api_keys_key_hash"), table_name="api_keys") + op.drop_index(op.f("ix_api_keys_user_id"), table_name="api_keys") + op.drop_index(op.f("ix_api_keys_id"), table_name="api_keys") + op.drop_table("api_keys") + op.drop_index(op.f("ix_users_supabase_id"), table_name="users") + op.drop_index(op.f("ix_users_id"), table_name="users") + op.drop_table("users") diff --git a/alembic/versions/005_add_trial_ends_at.py b/alembic/versions/005_add_trial_ends_at.py new file mode 100644 index 0000000..f42b2f1 --- /dev/null +++ b/alembic/versions/005_add_trial_ends_at.py @@ -0,0 +1,29 @@ +"""add trial_ends_at to users + +Revision ID: 005_add_trial_ends_at +Revises: 004_add_users_and_api_keys +Create Date: 2026-03-10 00:00:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "005_add_trial_ends_at" +down_revision: str | Sequence[str] | None = "004_add_users_and_api_keys" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Add trial_ends_at column to users table.""" + op.add_column("users", sa.Column("trial_ends_at", sa.DateTime(timezone=True), nullable=True)) + + +def downgrade() -> None: + """Remove trial_ends_at column from users table.""" + op.drop_column("users", "trial_ends_at") diff --git a/alembic/versions/006_unique_user_email.py b/alembic/versions/006_unique_user_email.py new file mode 100644 index 0000000..dd12485 --- /dev/null +++ b/alembic/versions/006_unique_user_email.py @@ -0,0 +1,27 @@ +"""add unique constraint on users.email + +Revision ID: 006_unique_user_email +Revises: 005_add_trial_ends_at +Create Date: 2026-03-10 00:00:00.000000 + +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "006_unique_user_email" +down_revision: str | Sequence[str] | None = "005_add_trial_ends_at" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Add UNIQUE constraint on users.email to prevent duplicate accounts.""" + op.create_unique_constraint("uq_users_email", "users", ["email"]) + + +def downgrade() -> None: + """Remove UNIQUE constraint on users.email.""" + op.drop_constraint("uq_users_email", "users", type_="unique") diff --git a/api/deps.py b/api/deps.py new file mode 100644 index 0000000..3e2708e --- /dev/null +++ b/api/deps.py @@ -0,0 +1,423 @@ +"""Dependency injection for authentication, authorization, and rate limiting.""" + +import hashlib +import logging +import time +from collections import defaultdict +from datetime import UTC, datetime, timedelta + +import jwt as pyjwt +from fastapi import Depends, Header, HTTPException, Request +from sqlalchemy import func, select, update +from sqlalchemy.orm import Session, joinedload + +from cherry_evals.config import settings +from db.postgres.base import get_db +from db.postgres.models import ApiKey, Collection, CollectionExample, User + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Tier limit constants +# --------------------------------------------------------------------------- + +FREE_LIMITS = { + "keyword_rpm": 30, + "semantic_searches_per_day": 50, + "llm_calls_per_day": 0, # blocked + "max_collections": 10, + "max_examples_per_collection": 1000, + "max_api_keys": 1, +} + +PRO_LIMITS = { + "keyword_rpm": 120, + "semantic_searches_per_day": -1, # unlimited + "llm_calls_per_day": 180, + "max_collections": -1, # unlimited + "max_examples_per_collection": -1, # unlimited + "max_api_keys": 10, +} + +ULTRA_LIMITS = { + "keyword_rpm": 120, + "semantic_searches_per_day": -1, # unlimited + "llm_calls_per_day": 300, # 50 Pro + 250 Flash combined + "max_collections": -1, # unlimited + "max_examples_per_collection": -1, # unlimited + "max_api_keys": 10, +} + + +def effective_tier(user: User) -> str: + """Return the effective tier, accounting for active trials.""" + if user.tier == "free" and user.trial_ends_at and user.trial_ends_at > datetime.now(UTC): + return "ultra" + return user.tier + + +def _get_limits(user: User | None) -> dict: + """Return tier limits for a user (or Free defaults for anonymous).""" + if user is None: + return FREE_LIMITS + tier = effective_tier(user) + if tier == "ultra": + return ULTRA_LIMITS + if tier == "pro": + return PRO_LIMITS + return FREE_LIMITS + + +# --------------------------------------------------------------------------- +# JWT verification (Supabase HS256) +# --------------------------------------------------------------------------- + + +def _decode_supabase_jwt(token: str) -> dict: + """Decode and verify a Supabase JWT using PyJWT (HS256). + + Validates signature, expiry, audience, and not-before claims. + Returns the JWT payload dict, or raises HTTPException on failure. + """ + try: + return pyjwt.decode( + token, + settings.supabase_jwt_secret, + algorithms=["HS256"], + audience="authenticated", + ) + except pyjwt.InvalidTokenError as e: + logger.warning("JWT decode failed: %s", e) + raise HTTPException(status_code=401, detail="Invalid token") from e + + +# --------------------------------------------------------------------------- +# User resolution helpers +# --------------------------------------------------------------------------- + + +def _provision_user(db: Session, supabase_id: str, email: str) -> User: + """JIT provision a user on first authenticated request.""" + user = User( + supabase_id=supabase_id, + email=email, + tier="free", + trial_ends_at=datetime.now(UTC) + timedelta(days=7), + quota_reset_at=datetime.now(UTC) + timedelta(days=1), + ) + db.add(user) + db.commit() + db.refresh(user) + logger.info("Provisioned new user: %s (%s)", supabase_id, email) + return user + + +def _resolve_from_jwt(token: str, db: Session) -> User: + """Resolve a User from a Bearer JWT.""" + payload = _decode_supabase_jwt(token) + supabase_id = payload.get("sub") + email = payload.get("email", "") + if not supabase_id: + raise HTTPException(status_code=401, detail="Invalid token: missing sub") + + user = db.execute(select(User).where(User.supabase_id == supabase_id)).scalar_one_or_none() + if not user: + user = _provision_user(db, supabase_id, email) + return user + + +def _resolve_from_api_key(raw_key: str, db: Session) -> User: + """Resolve a User from an API key.""" + key_hash = hashlib.sha256(raw_key.encode()).hexdigest() + api_key = db.execute( + select(ApiKey) + .options(joinedload(ApiKey.user)) + .where(ApiKey.key_hash == key_hash, ApiKey.is_active == True) # noqa: E712 + ).scalar_one_or_none() + + if not api_key: + raise HTTPException(status_code=401, detail="Invalid API key") + + # Update last_used_at + api_key.last_used_at = datetime.now(UTC) + db.commit() + + return api_key.user + + +# --------------------------------------------------------------------------- +# Core auth dependencies +# --------------------------------------------------------------------------- + + +def get_optional_user( + request: Request, + authorization: str | None = Header(default=None), + x_api_key: str | None = Header(default=None), + db: Session = Depends(get_db), +) -> User | None: + """Resolve the current user from JWT or API key, or return None. + + When auth_enabled is False, always returns None (open access). + """ + if not settings.auth_enabled: + return None + + # Try Bearer JWT first + if authorization and authorization.startswith("Bearer "): + token = authorization[7:] + return _resolve_from_jwt(token, db) + + # Try API key + if x_api_key: + return _resolve_from_api_key(x_api_key, db) + + return None + + +def get_current_user( + user: User | None = Depends(get_optional_user), +) -> User | None: + """Require an authenticated user (401 if not authed). + + When auth_enabled is False, returns None to allow open access. + """ + if not settings.auth_enabled: + return None + if user is None: + raise HTTPException(status_code=401, detail="Authentication required") + return user + + +def require_paid( + user: User | None = Depends(get_current_user), +) -> User | None: + """Require a paid-tier user — Pro, Ultra, or active trial (403 if Free). + + When auth_enabled is False, returns None. + """ + if not settings.auth_enabled: + return None + if user is None: + raise HTTPException(status_code=401, detail="Authentication required") + if effective_tier(user) not in ("pro", "ultra"): + raise HTTPException( + status_code=403, detail="This feature requires a Pro or Ultra subscription." + ) + return user + + +# --------------------------------------------------------------------------- +# Rate limiting (in-memory per-minute burst limiter) +# --------------------------------------------------------------------------- + +# {user_key: [(timestamp, ...)] } +_rate_limit_buckets: dict[str, list[float]] = defaultdict(list) + + +def _user_key(user: User | None, request: Request) -> str: + """Generate a rate limit key — user ID or client IP. + + WARNING: This rate limiter is per-process and not shared across workers. + For multi-worker deployments, use Redis-backed rate limiting. + """ + if user and hasattr(user, "supabase_id"): + return f"user:{user.supabase_id}" + # Prefer X-Forwarded-For from trusted proxy over request.client.host + forwarded = request.headers.get("x-forwarded-for", "") + host = ( + forwarded.split(",")[0].strip() + if forwarded + else (request.client.host if request.client else "unknown") + ) + return f"ip:{host}" + + +def check_search_rate_limit( + request: Request, + user: User | None = Depends(get_optional_user), +): + """Enforce per-minute rate limit on keyword search.""" + if not settings.auth_enabled: + return + + limits = _get_limits(user) + rpm = limits["keyword_rpm"] + key = _user_key(user, request) + + now = time.time() + window = now - 60 + bucket = _rate_limit_buckets[key] + # Prune old entries + _rate_limit_buckets[key] = [t for t in bucket if t > window] + bucket = _rate_limit_buckets[key] + + if len(bucket) >= rpm: + raise HTTPException(status_code=429, detail="Rate limit exceeded. Try again in a minute.") + + bucket.append(now) + + +# --------------------------------------------------------------------------- +# Daily quota checks (PostgreSQL-backed, survives restarts) +# --------------------------------------------------------------------------- + + +def _maybe_reset_quotas(user: User, db: Session): + """Reset daily counters if quota_reset_at is in the past.""" + now = datetime.now(UTC) + if user.quota_reset_at <= now: + user.llm_calls_today = 0 + user.semantic_searches_today = 0 + user.quota_reset_at = now + timedelta(days=1) + db.commit() + db.refresh(user) + + +def check_semantic_search_quota( + user: User | None = Depends(get_current_user), + db: Session = Depends(get_db), +): + """Check and increment daily semantic search quota.""" + if not settings.auth_enabled or user is None: + return + + _maybe_reset_quotas(user, db) + limits = _get_limits(user) + daily_limit = limits["semantic_searches_per_day"] + + if daily_limit == -1: + # Unlimited — increment without bound check + db.execute( + update(User) + .where(User.id == user.id) + .values(semantic_searches_today=User.semantic_searches_today + 1) + ) + else: + # Atomic check-and-increment: only succeeds if under limit + result = db.execute( + update(User) + .where(User.id == user.id, User.semantic_searches_today < daily_limit) + .values(semantic_searches_today=User.semantic_searches_today + 1) + ) + if result.rowcount == 0: + raise HTTPException( + status_code=429, + detail=f"Daily semantic search limit ({daily_limit}) reached. " + "Upgrade to Pro for unlimited.", + ) + db.commit() + + +def check_and_increment_llm_budget( + user: User | None = Depends(get_current_user), + db: Session = Depends(get_db), +): + """Check and increment daily LLM call quota.""" + if not settings.auth_enabled or user is None: + return + + _maybe_reset_quotas(user, db) + limits = _get_limits(user) + daily_limit = limits["llm_calls_per_day"] + + if daily_limit == 0: + raise HTTPException( + status_code=403, + detail="LLM-powered features require a Pro subscription.", + ) + + if daily_limit == -1: + # Unlimited — increment without bound check + db.execute( + update(User).where(User.id == user.id).values(llm_calls_today=User.llm_calls_today + 1) + ) + else: + # Atomic check-and-increment: only succeeds if under limit + result = db.execute( + update(User) + .where(User.id == user.id, User.llm_calls_today < daily_limit) + .values(llm_calls_today=User.llm_calls_today + 1) + ) + if result.rowcount == 0: + raise HTTPException( + status_code=429, + detail=f"Daily LLM call limit ({daily_limit}) reached.", + ) + db.commit() + + +# --------------------------------------------------------------------------- +# Collection / example limit checks +# --------------------------------------------------------------------------- + + +def check_collection_limit( + user: User | None = Depends(get_current_user), + db: Session = Depends(get_db), +): + """Check that Free users haven't exceeded their collection limit.""" + if not settings.auth_enabled or user is None: + return + + limits = _get_limits(user) + max_collections = limits["max_collections"] + if max_collections == -1: + return + + count = db.execute( + select(func.count(Collection.id)).where(Collection.user_id == user.supabase_id) + ).scalar() + + if count >= max_collections: + raise HTTPException( + status_code=403, + detail=f"Collection limit ({max_collections}) reached. Upgrade to Pro for unlimited.", + ) + + +def check_collection_example_limit( + collection_id: int, + user: User | None = Depends(get_current_user), + db: Session = Depends(get_db), +): + """Check that Free users haven't exceeded per-collection example limit. + + Also verifies the collection exists and belongs to the current user. + """ + if not settings.auth_enabled or user is None: + return + + # Verify collection ownership before checking limits + collection = db.get(Collection, collection_id) + if not collection: + raise HTTPException(status_code=404, detail="Collection not found") + if collection.user_id != user.supabase_id: + raise HTTPException(status_code=404, detail="Collection not found") + + limits = _get_limits(user) + max_examples = limits["max_examples_per_collection"] + if max_examples == -1: + return + + count = db.execute( + select(func.count(CollectionExample.id)).where( + CollectionExample.collection_id == collection_id + ) + ).scalar() + + if count >= max_examples: + raise HTTPException( + status_code=403, + detail=f"Example limit ({max_examples} per collection) reached. Upgrade to Pro.", + ) + + +def check_collection_ownership(collection: Collection, user: User | None): + """Verify collection belongs to the current user (when auth is enabled). + + Shared helper used by collections, export, and agents routes. + """ + if settings.auth_enabled and user is not None: + if collection.user_id != user.supabase_id: + raise HTTPException(status_code=404, detail="Collection not found") diff --git a/api/main.py b/api/main.py index 91a4a59..ae07a1f 100644 --- a/api/main.py +++ b/api/main.py @@ -1,8 +1,22 @@ """Cherry Evals FastAPI application.""" from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware -from api.routes import agents, analytics, collections, datasets, examples, export, health, search +from api.routes import ( + account, + agents, + analytics, + api_keys, + billing, + collections, + datasets, + examples, + export, + health, + search, +) +from cherry_evals.config import settings app = FastAPI( title="Cherry Evals", @@ -10,6 +24,15 @@ version="0.1.0", ) +# CORS — allow frontend origins +app.add_middleware( + CORSMiddleware, + allow_origins=[o.strip() for o in settings.cors_origins.split(",") if o.strip()], + allow_credentials=True, + allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], + allow_headers=["Authorization", "X-Api-Key", "X-Session-Id", "Content-Type"], +) + # Register routes app.include_router(health.router, tags=["health"]) app.include_router(datasets.router) @@ -19,6 +42,9 @@ app.include_router(export.router) app.include_router(analytics.router) app.include_router(agents.router) +app.include_router(billing.router) +app.include_router(api_keys.router) +app.include_router(account.router) @app.get("/") diff --git a/api/routes/account.py b/api/routes/account.py new file mode 100644 index 0000000..0492936 --- /dev/null +++ b/api/routes/account.py @@ -0,0 +1,179 @@ +"""Account and usage endpoints.""" + +import logging +from datetime import datetime + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import JSONResponse +from pydantic import BaseModel +from sqlalchemy import select +from sqlalchemy.orm import Session + +from api.deps import _get_limits, effective_tier, get_current_user +from db.postgres.base import get_db +from db.postgres.models import ApiKey, Collection, CollectionExample, CurationEvent, User + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/account", tags=["account"]) + + +class UsageStats(BaseModel): + """Current usage counters and limits.""" + + llm_calls_today: int + llm_calls_limit: int # -1 = unlimited, 0 = blocked + semantic_searches_today: int + semantic_searches_limit: int # -1 = unlimited + quota_resets_at: datetime + + +class AccountResponse(BaseModel): + """User profile + usage.""" + + email: str + tier: str + effective_tier: str + trial_ends_at: datetime | None + subscription_status: str | None + usage: UsageStats + created_at: datetime + + +@router.get("/me", response_model=AccountResponse) +def get_account( + user: User | None = Depends(get_current_user), + db: Session = Depends(get_db), +): + """Return current user profile and usage stats.""" + if user is None: + raise HTTPException(status_code=401, detail="Authentication required") + + limits = _get_limits(user) + + return AccountResponse( + email=user.email, + tier=user.tier, + effective_tier=effective_tier(user), + trial_ends_at=user.trial_ends_at, + subscription_status=user.subscription_status, + usage=UsageStats( + llm_calls_today=user.llm_calls_today, + llm_calls_limit=limits["llm_calls_per_day"], + semantic_searches_today=user.semantic_searches_today, + semantic_searches_limit=limits["semantic_searches_per_day"], + quota_resets_at=user.quota_reset_at, + ), + created_at=user.created_at, + ) + + +@router.get("/export") +def export_account_data( + user: User | None = Depends(get_current_user), + db: Session = Depends(get_db), +): + """GDPR Article 20 — data portability. Returns all user data as JSON.""" + if user is None: + raise HTTPException(status_code=401, detail="Authentication required") + + # Collections + collections = ( + db.execute(select(Collection).where(Collection.user_id == user.supabase_id)).scalars().all() + ) + collections_data = [] + for coll in collections: + example_ids = ( + db.execute( + select(CollectionExample.example_id).where( + CollectionExample.collection_id == coll.id + ) + ) + .scalars() + .all() + ) + collections_data.append( + { + "id": coll.id, + "name": coll.name, + "description": coll.description, + "example_ids": list(example_ids), + "created_at": coll.created_at.isoformat(), + } + ) + + # API keys (prefix only, never the hash) + api_keys = db.execute(select(ApiKey).where(ApiKey.user_id == user.id)).scalars().all() + keys_data = [ + {"prefix": k.key_prefix, "name": k.name, "created_at": k.created_at.isoformat()} + for k in api_keys + ] + + # Curation events + events = ( + db.execute( + select(CurationEvent) + .where(CurationEvent.user_id == user.supabase_id) + .order_by(CurationEvent.created_at.desc()) + .limit(10000) + ) + .scalars() + .all() + ) + events_data = [ + { + "event_type": ev.event_type, + "query": ev.query, + "example_id": ev.example_id, + "collection_id": ev.collection_id, + "created_at": ev.created_at.isoformat(), + } + for ev in events + ] + + return JSONResponse( + content={ + "account": { + "email": user.email, + "tier": user.tier, + "created_at": user.created_at.isoformat(), + "trial_ends_at": user.trial_ends_at.isoformat() if user.trial_ends_at else None, + }, + "collections": collections_data, + "api_keys": keys_data, + "curation_events": events_data, + } + ) + + +@router.delete("/me", status_code=204) +def delete_account( + user: User | None = Depends(get_current_user), + db: Session = Depends(get_db), +): + """GDPR Article 17 — right to erasure. Deletes the user and all associated data. + + Cascade deletes: API keys (via ORM cascade), collections (via user_id match), + and anonymises curation events (sets user_id to NULL). + """ + if user is None: + raise HTTPException(status_code=401, detail="Authentication required") + + # Anonymise curation events (preserve aggregated data, remove PII) + db.execute( + CurationEvent.__table__.update() + .where(CurationEvent.user_id == user.supabase_id) + .values(user_id=None, session_id=None) + ) + + # Delete user's collections (cascade removes collection_examples) + collections = ( + db.execute(select(Collection).where(Collection.user_id == user.supabase_id)).scalars().all() + ) + for coll in collections: + db.delete(coll) + + # Delete user (cascade removes api_keys via ORM relationship) + db.delete(user) + db.commit() + logger.info("Deleted account for user %s", user.supabase_id) diff --git a/api/routes/agents.py b/api/routes/agents.py index b0b1f6e..962783f 100644 --- a/api/routes/agents.py +++ b/api/routes/agents.py @@ -1,6 +1,7 @@ """Agent-powered API endpoints — LLM-driven ingestion, export, and discovery.""" import logging +import re from fastapi import APIRouter, Depends, Header, HTTPException from fastapi.responses import Response @@ -8,9 +9,15 @@ from sqlalchemy import select from sqlalchemy.orm import Session +from api.deps import ( + check_and_increment_llm_budget, + check_collection_ownership, + get_current_user, + require_paid, +) from core.traces.events import record_event from db.postgres.base import get_db -from db.postgres.models import Collection, CollectionExample, Dataset, Example +from db.postgres.models import Collection, CollectionExample, Dataset, Example, User logger = logging.getLogger(__name__) @@ -105,7 +112,11 @@ class CustomExportResponse(BaseModel): # --------------------------------------------------------------------------- -@router.post("/discover", response_model=DiscoverResponse) +@router.post( + "/discover", + response_model=DiscoverResponse, + dependencies=[Depends(require_paid), Depends(check_and_increment_llm_budget)], +) def discover_dataset(request: DiscoverDatasetRequest): """Discover a HuggingFace dataset matching a description. @@ -129,7 +140,11 @@ def discover_dataset(request: DiscoverDatasetRequest): ) -@router.post("/ingest", response_model=IngestResponse) +@router.post( + "/ingest", + response_model=IngestResponse, + dependencies=[Depends(require_paid), Depends(check_and_increment_llm_budget)], +) def ingest_dataset( request: IngestDatasetRequest, x_session_id: str | None = Header(default=None), @@ -160,12 +175,16 @@ def ingest_dataset( ) -@router.post("/{collection_id}/export-custom") +@router.post( + "/{collection_id}/export-custom", + dependencies=[Depends(require_paid), Depends(check_and_increment_llm_budget)], +) def export_collection_custom( collection_id: int, request: CustomExportRequest, db: Session = Depends(get_db), x_session_id: str | None = Header(default=None), + user: User | None = Depends(get_current_user), ): """Export a collection to any format using LLM-generated conversion logic. @@ -179,6 +198,8 @@ def export_collection_custom( if not collection: raise HTTPException(status_code=404, detail="Collection not found") + check_collection_ownership(collection, user) + examples = ( db.execute( select(Example) @@ -222,7 +243,8 @@ def export_collection_custom( ) # Return as downloadable file - filename = f"{collection.name.replace(' ', '_').lower()}{result.file_extension}" + safe_name = re.sub(r"[^a-zA-Z0-9_\-]", "_", collection.name).lower() + filename = f"{safe_name}{result.file_extension}" return Response( content=result.content, media_type=result.content_type, diff --git a/api/routes/api_keys.py b/api/routes/api_keys.py new file mode 100644 index 0000000..72c59d9 --- /dev/null +++ b/api/routes/api_keys.py @@ -0,0 +1,158 @@ +"""API key management endpoints.""" + +import hashlib +import logging +import secrets +from datetime import datetime + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from api.deps import _get_limits, get_current_user +from db.postgres.base import get_db +from db.postgres.models import ApiKey, User + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api-keys", tags=["api-keys"]) + +API_KEY_PREFIX = "ck_live_" + + +# --------------------------------------------------------------------------- +# Request / Response models +# --------------------------------------------------------------------------- + + +class CreateApiKeyRequest(BaseModel): + """Request to create a new API key.""" + + name: str = Field(default="Default", min_length=1, max_length=255) + + +class ApiKeyResponse(BaseModel): + """API key metadata (never includes the full key).""" + + id: int + key_prefix: str + name: str + last_used_at: datetime | None + is_active: bool + created_at: datetime + + +class ApiKeyCreatedResponse(ApiKeyResponse): + """Response when creating a key — includes the plaintext key exactly once.""" + + key: str + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.post("", response_model=ApiKeyCreatedResponse, status_code=201) +def create_api_key( + request: CreateApiKeyRequest, + user: User | None = Depends(get_current_user), + db: Session = Depends(get_db), +): + """Create a new API key. The plaintext key is returned only once.""" + if user is None: + raise HTTPException(status_code=401, detail="Authentication required") + + # Check tier limit + max_keys = _get_limits(user)["max_api_keys"] + active_count = db.execute( + select(func.count(ApiKey.id)).where( + ApiKey.user_id == user.id, + ApiKey.is_active == True, # noqa: E712 + ) + ).scalar() + if active_count >= max_keys: + raise HTTPException( + status_code=403, + detail=f"API key limit ({max_keys}) reached. Upgrade to Pro for more.", + ) + + # Generate key + raw_secret = secrets.token_urlsafe(24) + raw_key = f"{API_KEY_PREFIX}{raw_secret}" + key_hash = hashlib.sha256(raw_key.encode()).hexdigest() + key_prefix = raw_key[:12] + + api_key = ApiKey( + user_id=user.id, + key_prefix=key_prefix, + key_hash=key_hash, + name=request.name, + ) + db.add(api_key) + db.commit() + db.refresh(api_key) + + return ApiKeyCreatedResponse( + id=api_key.id, + key_prefix=api_key.key_prefix, + name=api_key.name, + last_used_at=api_key.last_used_at, + is_active=api_key.is_active, + created_at=api_key.created_at, + key=raw_key, + ) + + +@router.get("", response_model=list[ApiKeyResponse]) +def list_api_keys( + user: User | None = Depends(get_current_user), + db: Session = Depends(get_db), +): + """List all active API keys for the current user.""" + if user is None: + raise HTTPException(status_code=401, detail="Authentication required") + + keys = ( + db.execute( + select(ApiKey) + .where(ApiKey.user_id == user.id, ApiKey.is_active == True) # noqa: E712 + .order_by(ApiKey.created_at.desc()) + ) + .scalars() + .all() + ) + + return [ + ApiKeyResponse( + id=k.id, + key_prefix=k.key_prefix, + name=k.name, + last_used_at=k.last_used_at, + is_active=k.is_active, + created_at=k.created_at, + ) + for k in keys + ] + + +@router.delete("/{key_id}", status_code=204) +def revoke_api_key( + key_id: int, + user: User | None = Depends(get_current_user), + db: Session = Depends(get_db), +): + """Revoke (soft-delete) an API key.""" + if user is None: + raise HTTPException(status_code=401, detail="Authentication required") + + api_key = db.execute( + select(ApiKey).where(ApiKey.id == key_id, ApiKey.user_id == user.id) + ).scalar_one_or_none() + + if not api_key: + raise HTTPException(status_code=404, detail="API key not found") + + api_key.is_active = False + db.commit() diff --git a/api/routes/billing.py b/api/routes/billing.py new file mode 100644 index 0000000..d8a56e1 --- /dev/null +++ b/api/routes/billing.py @@ -0,0 +1,153 @@ +"""Polar.sh billing webhook endpoint.""" + +import base64 +import hashlib +import hmac +import logging + +from fastapi import APIRouter, Depends, HTTPException, Request +from sqlalchemy import select +from sqlalchemy.orm import Session + +from cherry_evals.config import settings +from db.postgres.base import get_db +from db.postgres.models import User + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["billing"]) + + +def _verify_polar_signature(payload: bytes, headers: dict[str, str]) -> bool: + """Verify Polar webhook signature (Svix format). + + Polar uses Svix under the hood. Signature format: + - Headers: webhook-id, webhook-timestamp, webhook-signature + - Signed content: ``{msg_id}.{timestamp}.{body}`` + - Secret may have ``whsec_`` prefix (base64-encoded key) + - Signature header: ``v1,`` + """ + if not settings.polar_webhook_secret: + logger.warning("POLAR_WEBHOOK_SECRET not configured, rejecting webhook") + return False + + msg_id = headers.get("webhook-id", "") + timestamp = headers.get("webhook-timestamp", "") + signature = headers.get("webhook-signature", "") + + if not all([msg_id, timestamp, signature]): + return False + + # Svix secret format: whsec_ + secret = settings.polar_webhook_secret + if secret.startswith("whsec_"): + secret = secret[6:] + + try: + key = base64.b64decode(secret) + except Exception: + key = secret.encode() + + to_sign = f"{msg_id}.{timestamp}.".encode() + payload + expected = base64.b64encode(hmac.new(key, to_sign, hashlib.sha256).digest()).decode() + + # Signature header may contain multiple versions: "v1, v1," + for sig_part in signature.split(): + if sig_part.startswith("v1,"): + if hmac.compare_digest(expected, sig_part[3:]): + return True + + return False + + +@router.post("/webhooks/polar") +async def polar_webhook(request: Request, db: Session = Depends(get_db)): + """Handle Polar.sh subscription lifecycle webhooks. + + Events handled: + - subscription.created / subscription.updated → set tier to pro + - subscription.canceled / subscription.revoked → set tier to free + """ + body = await request.body() + + if not _verify_polar_signature(body, dict(request.headers)): + raise HTTPException(status_code=400, detail="Invalid webhook signature") + + import json + + try: + event = json.loads(body) + except json.JSONDecodeError: + raise HTTPException(status_code=400, detail="Invalid JSON payload") + + event_type = event.get("type", "") + data = event.get("data", {}) + + # Extract customer email from the nested Polar event structure + customer = data.get("customer", {}) + customer_email = customer.get("email", "") + customer_id = str(customer.get("id", "")) + + if not customer_email: + logger.warning("Polar webhook missing customer email: %s", event_type) + return {"status": "ignored", "reason": "no customer email"} + + # Find user — prefer polar_customer_id for returning subscribers, fall back to email + user = None + if customer_id: + user = db.execute( + select(User).where(User.polar_customer_id == customer_id) + ).scalar_one_or_none() + if not user: + user = db.execute(select(User).where(User.email == customer_email)).scalar_one_or_none() + if not user: + logger.warning("Polar webhook for unknown user: %s", customer_email) + return {"status": "ignored", "reason": "user not found"} + + subscription_id = str(data.get("id", "")) + subscription_status = data.get("status", "") + + if event_type in ("subscription.created", "subscription.updated"): + # Determine tier from product ID + product = data.get("product", {}) + product_id = str(product.get("id", "")) if isinstance(product, dict) else "" + if not product_id: + product_id = str(data.get("product_id", "")) + + if product_id and product_id == settings.polar_ultra_product_id: + tier = "ultra" + elif product_id and product_id == settings.polar_pro_product_id: + tier = "pro" + else: + # Default to pro if product ID not configured yet + tier = "pro" + + # Only upgrade tier when subscription is confirmed active + if subscription_status in ("active", "trialing"): + user.tier = tier + # Trial is superseded by paid subscription + user.trial_ends_at = None + logger.info( + "Upgraded user %s to %s (subscription %s)", user.email, tier, subscription_id + ) + else: + logger.info( + "Subscription %s for %s has status %s — not upgrading tier yet", + subscription_id, + user.email, + subscription_status, + ) + + user.polar_customer_id = customer_id + user.polar_subscription_id = subscription_id + user.subscription_status = subscription_status + elif event_type in ("subscription.canceled", "subscription.revoked"): + user.tier = "free" + user.subscription_status = subscription_status + logger.info("Downgraded user %s to free (subscription %s)", user.email, subscription_id) + else: + logger.info("Ignoring Polar event type: %s", event_type) + return {"status": "ignored", "reason": f"unhandled event type: {event_type}"} + + db.commit() + return {"status": "ok"} diff --git a/api/routes/collections.py b/api/routes/collections.py index 5d65f2d..2dc6047 100644 --- a/api/routes/collections.py +++ b/api/routes/collections.py @@ -6,6 +6,12 @@ from sqlalchemy import func, select from sqlalchemy.orm import Session +from api.deps import ( + check_collection_example_limit, + check_collection_limit, + check_collection_ownership, + get_current_user, +) from api.models.collections import ( AddExamplesRequest, CollectionCreate, @@ -16,9 +22,10 @@ CollectionUpdate, RemoveExamplesRequest, ) +from cherry_evals.config import settings from core.traces.events import record_event from db.postgres.base import get_db -from db.postgres.models import Collection, CollectionExample, Example +from db.postgres.models import Collection, CollectionExample, Example, User logger = logging.getLogger(__name__) @@ -44,10 +51,23 @@ def _collection_to_response(db: Session, collection: Collection) -> CollectionRe ) -@router.post("", response_model=CollectionResponse, status_code=201) -def create_collection(request: CollectionCreate, db: Session = Depends(get_db)): +@router.post( + "", + response_model=CollectionResponse, + status_code=201, + dependencies=[Depends(check_collection_limit)], +) +def create_collection( + request: CollectionCreate, + db: Session = Depends(get_db), + user: User | None = Depends(get_current_user), +): """Create a new collection.""" - collection = Collection(name=request.name, description=request.description) + collection = Collection( + name=request.name, + description=request.description, + user_id=user.supabase_id if user else None, + ) db.add(collection) db.commit() db.refresh(collection) @@ -55,11 +75,15 @@ def create_collection(request: CollectionCreate, db: Session = Depends(get_db)): @router.get("", response_model=CollectionListResponse) -def list_collections(db: Session = Depends(get_db)): - """List all collections.""" - collections = ( - db.execute(select(Collection).order_by(Collection.created_at.desc())).scalars().all() - ) +def list_collections( + db: Session = Depends(get_db), + user: User | None = Depends(get_current_user), +): + """List collections owned by the current user.""" + query = select(Collection).order_by(Collection.created_at.desc()) + if settings.auth_enabled and user is not None: + query = query.where(Collection.user_id == user.supabase_id) + collections = db.execute(query).scalars().all() return CollectionListResponse( collections=[_collection_to_response(db, c) for c in collections], @@ -68,20 +92,31 @@ def list_collections(db: Session = Depends(get_db)): @router.get("/{collection_id}", response_model=CollectionResponse) -def get_collection(collection_id: int, db: Session = Depends(get_db)): +def get_collection( + collection_id: int, + db: Session = Depends(get_db), + user: User | None = Depends(get_current_user), +): """Get a single collection by ID.""" collection = db.get(Collection, collection_id) if not collection: raise HTTPException(status_code=404, detail="Collection not found") + check_collection_ownership(collection, user) return _collection_to_response(db, collection) @router.put("/{collection_id}", response_model=CollectionResponse) -def update_collection(collection_id: int, request: CollectionUpdate, db: Session = Depends(get_db)): +def update_collection( + collection_id: int, + request: CollectionUpdate, + db: Session = Depends(get_db), + user: User | None = Depends(get_current_user), +): """Update collection metadata.""" collection = db.get(Collection, collection_id) if not collection: raise HTTPException(status_code=404, detail="Collection not found") + check_collection_ownership(collection, user) if request.name is not None: collection.name = request.name @@ -94,22 +129,32 @@ def update_collection(collection_id: int, request: CollectionUpdate, db: Session @router.delete("/{collection_id}", status_code=204) -def delete_collection(collection_id: int, db: Session = Depends(get_db)): +def delete_collection( + collection_id: int, + db: Session = Depends(get_db), + user: User | None = Depends(get_current_user), +): """Delete a collection and all its example associations.""" collection = db.get(Collection, collection_id) if not collection: raise HTTPException(status_code=404, detail="Collection not found") + check_collection_ownership(collection, user) db.delete(collection) db.commit() @router.get("/{collection_id}/examples", response_model=CollectionExamplesListResponse) -def list_collection_examples(collection_id: int, db: Session = Depends(get_db)): +def list_collection_examples( + collection_id: int, + db: Session = Depends(get_db), + user: User | None = Depends(get_current_user), +): """List all examples in a collection.""" collection = db.get(Collection, collection_id) if not collection: raise HTTPException(status_code=404, detail="Collection not found") + check_collection_ownership(collection, user) rows = db.execute( select(Example, CollectionExample.added_at) @@ -145,11 +190,14 @@ def add_examples( request: AddExamplesRequest, db: Session = Depends(get_db), x_session_id: str | None = Header(default=None), + user: User | None = Depends(get_current_user), + _limit: None = Depends(check_collection_example_limit), ): """Add examples to a collection by ID. Skips duplicates.""" collection = db.get(Collection, collection_id) if not collection: raise HTTPException(status_code=404, detail="Collection not found") + check_collection_ownership(collection, user) # Check which examples already exist in collection existing = set( @@ -198,8 +246,15 @@ def remove_example( example_id: int, db: Session = Depends(get_db), x_session_id: str | None = Header(default=None), + user: User | None = Depends(get_current_user), ): """Remove a single example from a collection.""" + # Ownership check — prevent IDOR + collection = db.get(Collection, collection_id) + if not collection: + raise HTTPException(status_code=404, detail="Collection not found") + check_collection_ownership(collection, user) + ce = db.execute( select(CollectionExample).where( CollectionExample.collection_id == collection_id, @@ -228,12 +283,16 @@ def remove_example( @router.post("/{collection_id}/examples/bulk-remove", status_code=200) def bulk_remove_examples( - collection_id: int, request: RemoveExamplesRequest, db: Session = Depends(get_db) + collection_id: int, + request: RemoveExamplesRequest, + db: Session = Depends(get_db), + user: User | None = Depends(get_current_user), ): """Remove multiple examples from a collection.""" collection = db.get(Collection, collection_id) if not collection: raise HTTPException(status_code=404, detail="Collection not found") + check_collection_ownership(collection, user) ces = ( db.execute( diff --git a/api/routes/export.py b/api/routes/export.py index 483bccc..bc40cf3 100644 --- a/api/routes/export.py +++ b/api/routes/export.py @@ -1,18 +1,21 @@ """Export API endpoints.""" import logging +import re from fastapi import APIRouter, Depends, Header, HTTPException from fastapi.responses import Response from sqlalchemy import select from sqlalchemy.orm import Session +from api.deps import check_collection_ownership, effective_tier, get_current_user from api.models.export import ExportFormat, ExportRequest, LangfuseExportResponse +from cherry_evals.config import settings from core.export.formats import to_csv, to_json, to_jsonl from core.export.langfuse_export import LangfuseExportError, export_to_langfuse from core.traces.events import record_event from db.postgres.base import get_db -from db.postgres.models import Collection, CollectionExample, Dataset, Example +from db.postgres.models import Collection, CollectionExample, Dataset, Example, User logger = logging.getLogger(__name__) @@ -31,6 +34,11 @@ } +def _sanitize_filename(name: str) -> str: + """Sanitize a collection name for use in Content-Disposition header.""" + return re.sub(r"[^a-zA-Z0-9_\-]", "_", name).lower() + + def _get_collection_examples(db: Session, collection_id: int): """Fetch all examples in a collection with their dataset names.""" rows = ( @@ -60,19 +68,30 @@ def export_collection( request: ExportRequest, db: Session = Depends(get_db), x_session_id: str | None = Header(default=None), + user: User | None = Depends(get_current_user), ): """Export a collection to the specified format. For json/jsonl/csv: returns the file as a download. - For langfuse: pushes to Langfuse and returns a summary. + For langfuse: pushes to Langfuse and returns a summary (requires paid tier). """ collection = db.get(Collection, collection_id) if not collection: raise HTTPException(status_code=404, detail="Collection not found") + check_collection_ownership(collection, user) + examples, dataset_names = _get_collection_examples(db, collection_id) if request.format == ExportFormat.langfuse: + # Langfuse is a premium integration — require paid tier + if settings.auth_enabled and user is not None: + if effective_tier(user) not in ("pro", "ultra"): + raise HTTPException( + status_code=403, + detail="Langfuse export requires a Pro or Ultra subscription.", + ) + ds_name = request.langfuse_dataset_name or collection.name try: result = export_to_langfuse( @@ -81,8 +100,8 @@ def export_collection( dataset_description=collection.description, dataset_names=dataset_names, ) - except LangfuseExportError as e: - raise HTTPException(status_code=502, detail=str(e)) + except LangfuseExportError: + raise HTTPException(status_code=502, detail="Langfuse export failed") try: record_event( @@ -106,7 +125,7 @@ def export_collection( content = converters[request.format](examples, dataset_names) media_type = _FORMAT_MEDIA_TYPES[request.format] ext = _FORMAT_EXTENSIONS[request.format] - filename = f"{collection.name.replace(' ', '_').lower()}.{ext}" + filename = f"{_sanitize_filename(collection.name)}.{ext}" try: record_event( diff --git a/api/routes/search.py b/api/routes/search.py index a524952..2f406e6 100644 --- a/api/routes/search.py +++ b/api/routes/search.py @@ -5,6 +5,12 @@ from fastapi import APIRouter, Depends, Header, HTTPException from sqlalchemy.orm import Session +from api.deps import ( + check_and_increment_llm_budget, + check_search_rate_limit, + check_semantic_search_quota, + require_paid, +) from api.models.search import ( FacetRequest, FacetResponse, @@ -26,7 +32,11 @@ router = APIRouter(prefix="/search", tags=["search"]) -@router.post("", response_model=SearchResponse) +@router.post( + "", + response_model=SearchResponse, + dependencies=[Depends(check_search_rate_limit)], +) def search( request: SearchRequest, db: Session = Depends(get_db), @@ -68,7 +78,11 @@ def search( ) -@router.post("/semantic", response_model=SearchResponse) +@router.post( + "/semantic", + response_model=SearchResponse, + dependencies=[Depends(check_semantic_search_quota)], +) def search_semantic( request: SemanticSearchRequest, x_session_id: str | None = Header(default=None), @@ -89,10 +103,10 @@ def search_semantic( subject=request.subject, ) except Exception as e: - logger.warning(f"Semantic search failed: {e}") + logger.warning("Semantic search failed: %s", e) raise HTTPException( status_code=503, - detail=f"Semantic search unavailable: {e}", + detail="Semantic search is temporarily unavailable.", ) return SearchResponse( @@ -132,7 +146,7 @@ def search_hybrid( ) except Exception as e: # Fall back to keyword search if semantic/hybrid fails - logger.warning(f"Hybrid search failed, falling back to keyword: {e}") + logger.warning("Hybrid search failed, falling back to keyword: %s", e) kw_results, total = keyword_search( db=db, query=request.query, @@ -164,7 +178,11 @@ def search_hybrid( ) -@router.post("/intelligent", response_model=IntelligentSearchResponse) +@router.post( + "/intelligent", + response_model=IntelligentSearchResponse, + dependencies=[Depends(require_paid), Depends(check_and_increment_llm_budget)], +) def search_intelligent( request: IntelligentSearchRequest, db: Session = Depends(get_db), diff --git a/cherry_evals/config.py b/cherry_evals/config.py index 0a77046..868a8a9 100644 --- a/cherry_evals/config.py +++ b/cherry_evals/config.py @@ -1,9 +1,13 @@ """Configuration for Cherry Evals using pydantic-settings.""" +import logging from pathlib import Path +from pydantic import model_validator from pydantic_settings import BaseSettings, SettingsConfigDict +_config_logger = logging.getLogger(__name__) + class Settings(BaseSettings): """Application settings.""" @@ -36,9 +40,40 @@ class Settings(BaseSettings): langfuse_secret_key: str = "" langfuse_base_url: str = "https://cloud.langfuse.com" + # Auth (Supabase) + supabase_url: str = "" + supabase_jwt_secret: str = "" + + # Billing (Polar.sh) + polar_webhook_secret: str = "" + polar_pro_product_id: str = "" + polar_ultra_product_id: str = "" + + # Auth toggle (set False for local dev / tests) + auth_enabled: bool = True + + # CORS origins (comma-separated in env, parsed below) + cors_origins: str = "https://app.cherryevals.com,http://localhost:5173" + # Optional overrides cherry_data_dir: Path = Path("./data") cherry_log_level: str = "INFO" + @model_validator(mode="after") + def _check_auth_secrets(self) -> "Settings": + """Disable auth if JWT secret is missing (safe dev default). + + In production, SUPABASE_JWT_SECRET must be set when AUTH_ENABLED=True. + Locally this falls back to open access with a loud warning. + """ + if self.auth_enabled and not self.supabase_jwt_secret: + _config_logger.warning( + "AUTH_ENABLED=True but SUPABASE_JWT_SECRET is empty — " + "falling back to AUTH_ENABLED=False. " + "Set SUPABASE_JWT_SECRET for production." + ) + self.auth_enabled = False + return self + settings = Settings() diff --git a/db/postgres/models.py b/db/postgres/models.py index 76d977b..8114aeb 100644 --- a/db/postgres/models.py +++ b/db/postgres/models.py @@ -3,12 +3,83 @@ from datetime import datetime from typing import Any -from sqlalchemy import JSON, DateTime, Float, ForeignKey, Index, Integer, String, Text, func +from sqlalchemy import ( + JSON, + Boolean, + DateTime, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + func, +) from sqlalchemy.orm import Mapped, mapped_column, relationship from db.postgres.base import Base +class User(Base): + """Authenticated user, provisioned JIT from Supabase JWT or API key.""" + + __tablename__ = "users" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) + supabase_id: Mapped[str] = mapped_column(String(255), unique=True, nullable=False, index=True) + email: Mapped[str] = mapped_column(String(255), unique=True, nullable=False) + tier: Mapped[str] = mapped_column(String(50), nullable=False, default="free") + + # Trial + trial_ends_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + # Polar.sh subscription + polar_customer_id: Mapped[str | None] = mapped_column(String(255), nullable=True) + polar_subscription_id: Mapped[str | None] = mapped_column(String(255), nullable=True) + subscription_status: Mapped[str | None] = mapped_column(String(50), nullable=True) + + # Daily usage counters (reset at quota_reset_at) + llm_calls_today: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + semantic_searches_today: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + quota_reset_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False + ) + + # Relationships + api_keys: Mapped[list["ApiKey"]] = relationship( + "ApiKey", back_populates="user", cascade="all, delete-orphan" + ) + + +class ApiKey(Base): + """API key for programmatic access.""" + + __tablename__ = "api_keys" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) + user_id: Mapped[int] = mapped_column( + Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + key_prefix: Mapped[str] = mapped_column(String(20), nullable=False) + key_hash: Mapped[str] = mapped_column(String(64), unique=True, nullable=False) + name: Mapped[str] = mapped_column(String(255), nullable=False, default="Default") + last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + # Relationships + user: Mapped["User"] = relationship("User", back_populates="api_keys") + + class Dataset(Base): """Dataset model.""" diff --git a/frontend/package-lock.json b/frontend/package-lock.json index bd14910..8bd2b8f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,6 +8,7 @@ "name": "frontend", "version": "0.0.0", "dependencies": { + "@supabase/supabase-js": "^2.99.0", "react": "^19.2.0", "react-dom": "^19.2.0", "react-router-dom": "^7.13.1" @@ -1367,6 +1368,86 @@ "win32" ] }, + "node_modules/@supabase/auth-js": { + "version": "2.99.0", + "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.99.0.tgz", + "integrity": "sha512-tHiIST/OEoLmWBE+3X69xRY5srJM/lL86KltmMlIfDo9ePJLo14vQQV9T4NF+P+MoGhCwQL1GTmk51zuAFMXKw==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@supabase/functions-js": { + "version": "2.99.0", + "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.99.0.tgz", + "integrity": "sha512-zA9oad6EqGwMLLu2LfP1bXbqKcJGiotAdbdTfZG7YS7619YZQAEgejj9mp+E5vglKE1yMWbKK+S1J3PbuUtgLg==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@supabase/postgrest-js": { + "version": "2.99.0", + "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-2.99.0.tgz", + "integrity": "sha512-8qfOMi2pu9y0IQhUAeFqjrvR49G4ELGevXCWV9qAHXFQ/h2FFh0I8PYjFQj4rHcHSq6hrpozDnS1vbQU8NAQ/A==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@supabase/realtime-js": { + "version": "2.99.0", + "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.99.0.tgz", + "integrity": "sha512-7nFTZhNeANR7FvEY6PfWLCfE8dHqcaJd9SuR7IPEZvBPG9K4uEHMivpjZx4NWRSU7Eji7ZbKy2LG+cJ48DhwHg==", + "license": "MIT", + "dependencies": { + "@types/phoenix": "^1.6.6", + "@types/ws": "^8.18.1", + "tslib": "2.8.1", + "ws": "^8.18.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@supabase/storage-js": { + "version": "2.99.0", + "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.99.0.tgz", + "integrity": "sha512-mAEEbfsght5EEALejYrwAP9k8sFBGjfMZT8n4SyMXk2iYuWVeRMs1kA/uKg0uDMctWdZ0bL+L4jZzksUJpCjMA==", + "license": "MIT", + "dependencies": { + "iceberg-js": "^0.8.1", + "tslib": "2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@supabase/supabase-js": { + "version": "2.99.0", + "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.99.0.tgz", + "integrity": "sha512-SP9Sn9tsHDB7N4u2gT13rdeZJewE4xibAxasG7vOz+fYi92+XkMMbWNx0uGK53zKTnAnvTs16isRooyBy4sn5w==", + "license": "MIT", + "dependencies": { + "@supabase/auth-js": "2.99.0", + "@supabase/functions-js": "2.99.0", + "@supabase/postgrest-js": "2.99.0", + "@supabase/realtime-js": "2.99.0", + "@supabase/storage-js": "2.99.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@tailwindcss/node": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.2.1.tgz", @@ -1698,6 +1779,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/node": { + "version": "25.4.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.4.0.tgz", + "integrity": "sha512-9wLpoeWuBlcbBpOY3XmzSTG3oscB6xjBEEtn+pYXTfhyXhIxC5FsBer2KTopBlvKEiW9l13po9fq+SJY/5lkhw==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/phoenix": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/@types/phoenix/-/phoenix-1.6.7.tgz", + "integrity": "sha512-oN9ive//QSBkf19rfDv45M7eZPi0eEXylht2OLEXicu5b4KoQ1OzXIw+xDSGWxSxe1JmepRR/ZH283vsu518/Q==", + "license": "MIT" + }, "node_modules/@types/react": { "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", @@ -1719,6 +1815,15 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@vitejs/plugin-react": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.1.4.tgz", @@ -2468,6 +2573,15 @@ "hermes-estree": "0.25.1" } }, + "node_modules/iceberg-js": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/iceberg-js/-/iceberg-js-0.8.1.tgz", + "integrity": "sha512-1dhVQZXhcHje7798IVM+xoo/1ZdVfzOMIc8/rgVSijRK38EDqOJoGula9N/8ZI5RD8QTxNQtK/Gozpr+qUqRRA==", + "license": "MIT", + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -2541,7 +2655,6 @@ "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "jiti": "lib/jiti-cli.mjs" } @@ -2643,7 +2756,6 @@ "integrity": "sha512-l51N2r93WmGUye3WuFoN5k10zyvrVs0qfKBhyC5ogUQ6Ew6JUSswh78mbSO+IU3nTWsyOArqPCcShdQSadghBQ==", "dev": true, "license": "MPL-2.0", - "peer": true, "dependencies": { "detect-libc": "^2.0.3" }, @@ -3393,6 +3505,12 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -3406,6 +3524,12 @@ "node": ">= 0.8.0" } }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + }, "node_modules/update-browserslist-db": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", @@ -3549,6 +3673,27 @@ "node": ">=0.10.0" } }, + "node_modules/ws": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", + "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index e2ebc0d..f68447f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,6 +10,7 @@ "preview": "vite preview" }, "dependencies": { + "@supabase/supabase-js": "^2.99.0", "react": "^19.2.0", "react-dom": "^19.2.0", "react-router-dom": "^7.13.1" diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 0320133..4614892 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1,21 +1,58 @@ import { BrowserRouter, Routes, Route } from 'react-router-dom'; +import { AuthProvider } from './lib/AuthContext'; import Layout from './components/Layout'; +import ProtectedRoute from './components/ProtectedRoute'; import SearchPage from './pages/SearchPage'; import DatasetsPage from './pages/DatasetsPage'; import CollectionsPage from './pages/CollectionsPage'; import CollectionDetailPage from './pages/CollectionDetailPage'; +import LoginPage from './pages/LoginPage'; +import PricingPage from './pages/PricingPage'; +import AccountPage from './pages/AccountPage'; +import PrivacyPage from './pages/PrivacyPage'; +import TermsPage from './pages/TermsPage'; +import CookiePage from './pages/CookiePage'; export default function App() { return ( - - - } /> - } /> - } /> - } /> - - + + + + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + + + } + /> + + + + } + /> + + + + } + /> + + + ); } diff --git a/frontend/src/components/Layout.jsx b/frontend/src/components/Layout.jsx index 12090e2..ab1485b 100644 --- a/frontend/src/components/Layout.jsx +++ b/frontend/src/components/Layout.jsx @@ -1,12 +1,16 @@ -import { NavLink } from 'react-router-dom'; +import { NavLink, Link } from 'react-router-dom'; +import { useAuth } from '../lib/AuthContext'; const navItems = [ { path: '/', label: 'Search' }, { path: '/datasets', label: 'Datasets' }, { path: '/collections', label: 'Collections' }, + { path: '/pricing', label: 'Pricing' }, ]; export default function Layout({ children }) { + const { user, loading } = useAuth(); + return (
{children}
+
+
+ Elastic License 2.0 +
+ Terms + Privacy + Cookies +
+
+
); } diff --git a/frontend/src/components/ProtectedRoute.jsx b/frontend/src/components/ProtectedRoute.jsx new file mode 100644 index 0000000..c494e1b --- /dev/null +++ b/frontend/src/components/ProtectedRoute.jsx @@ -0,0 +1,20 @@ +import { Navigate } from 'react-router-dom'; +import { useAuth } from '../lib/AuthContext'; +import { supabase } from '../lib/supabase'; + +export default function ProtectedRoute({ children }) { + const { user, loading } = useAuth(); + + // If Supabase isn't configured, allow access (dev mode) + if (!supabase) return children; + + if (loading) { + return
Loading...
; + } + + if (!user) { + return ; + } + + return children; +} diff --git a/frontend/src/lib/AuthContext.jsx b/frontend/src/lib/AuthContext.jsx new file mode 100644 index 0000000..8205e94 --- /dev/null +++ b/frontend/src/lib/AuthContext.jsx @@ -0,0 +1,57 @@ +import { createContext, useContext, useEffect, useState } from 'react'; +import { supabase } from './supabase'; + +const AuthContext = createContext({ + user: null, + session: null, + loading: true, + signOut: async () => {}, +}); + +export function AuthProvider({ children }) { + const [user, setUser] = useState(null); + const [session, setSession] = useState(null); + const [loading, setLoading] = useState(true); + + useEffect(() => { + if (!supabase) { + setLoading(false); + return; + } + + // Get initial session + supabase.auth.getSession().then(({ data: { session } }) => { + setSession(session); + setUser(session?.user ?? null); + setLoading(false); + }); + + // Listen for auth changes + const { + data: { subscription }, + } = supabase.auth.onAuthStateChange((_event, session) => { + setSession(session); + setUser(session?.user ?? null); + }); + + return () => subscription.unsubscribe(); + }, []); + + const signOut = async () => { + if (supabase) { + await supabase.auth.signOut(); + } + setUser(null); + setSession(null); + }; + + return ( + + {children} + + ); +} + +export function useAuth() { + return useContext(AuthContext); +} diff --git a/frontend/src/lib/api.js b/frontend/src/lib/api.js index b16c4ee..9ddaee3 100644 --- a/frontend/src/lib/api.js +++ b/frontend/src/lib/api.js @@ -1,8 +1,24 @@ -const BASE = '/api'; +import { supabase } from './supabase'; + +// In production, VITE_API_BASE_URL points to Cloud Run directly. +// In dev, Vite proxies /api to localhost:8000. +const BASE = import.meta.env.VITE_API_BASE_URL || '/api'; async function request(path, options = {}) { + const authHeaders = {}; + if (supabase) { + const { data } = await supabase.auth.getSession(); + if (data.session?.access_token) { + authHeaders['Authorization'] = `Bearer ${data.session.access_token}`; + } + } + const res = await fetch(`${BASE}${path}`, { - headers: { 'Content-Type': 'application/json', ...options.headers }, + headers: { + 'Content-Type': 'application/json', + ...authHeaders, + ...options.headers, + }, ...options, }); if (!res.ok) { @@ -88,3 +104,16 @@ export const exportCollection = (id, format) => method: 'POST', body: JSON.stringify({ format }), }); + +// Account +export const getAccount = () => request('/account/me'); + +// API Keys +export const listApiKeys = () => request('/api-keys'); +export const createApiKey = (name) => + request('/api-keys', { + method: 'POST', + body: JSON.stringify({ name }), + }); +export const revokeApiKey = (id) => + request(`/api-keys/${id}`, { method: 'DELETE' }); diff --git a/frontend/src/lib/supabase.js b/frontend/src/lib/supabase.js new file mode 100644 index 0000000..dd425e3 --- /dev/null +++ b/frontend/src/lib/supabase.js @@ -0,0 +1,10 @@ +import { createClient } from '@supabase/supabase-js'; + +const supabaseUrl = import.meta.env.VITE_SUPABASE_URL || ''; +const supabaseAnonKey = import.meta.env.VITE_SUPABASE_ANON_KEY || ''; + +// Graceful degradation: if not configured, supabase will be null +export const supabase = + supabaseUrl && supabaseAnonKey + ? createClient(supabaseUrl, supabaseAnonKey) + : null; diff --git a/frontend/src/pages/AccountPage.jsx b/frontend/src/pages/AccountPage.jsx new file mode 100644 index 0000000..d9d6c01 --- /dev/null +++ b/frontend/src/pages/AccountPage.jsx @@ -0,0 +1,233 @@ +import { useEffect, useState } from 'react'; +import { useAuth } from '../lib/AuthContext'; +import { getAccount, listApiKeys, createApiKey, revokeApiKey } from '../lib/api'; + +const TIER_BADGE = { + free: 'bg-gray-100 text-gray-600', + pro: 'bg-red-100 text-red-700', + ultra: 'bg-purple-100 text-purple-700', +}; + +function trialDaysLeft(trialEndsAt) { + if (!trialEndsAt) return 0; + const diff = new Date(trialEndsAt) - new Date(); + return Math.max(0, Math.ceil(diff / (1000 * 60 * 60 * 24))); +} + +export default function AccountPage() { + const { user, signOut } = useAuth(); + const [account, setAccount] = useState(null); + const [apiKeys, setApiKeys] = useState([]); + const [newKeyName, setNewKeyName] = useState(''); + const [createdKey, setCreatedKey] = useState(null); + const [error, setError] = useState(''); + const [loading, setLoading] = useState(true); + + useEffect(() => { + Promise.all([ + getAccount().catch(() => null), + listApiKeys().catch(() => []), + ]).then(([acc, keys]) => { + setAccount(acc); + setApiKeys(Array.isArray(keys) ? keys : []); + setLoading(false); + }); + }, []); + + const handleCreateKey = async (e) => { + e.preventDefault(); + setError(''); + setCreatedKey(null); + try { + const result = await createApiKey(newKeyName || 'Default'); + setCreatedKey(result.key); + setNewKeyName(''); + // Refresh list + const keys = await listApiKeys(); + setApiKeys(Array.isArray(keys) ? keys : []); + } catch (err) { + setError(err.message); + } + }; + + const handleRevoke = async (id) => { + try { + await revokeApiKey(id); + setApiKeys(apiKeys.filter((k) => k.id !== id)); + } catch (err) { + setError(err.message); + } + }; + + if (loading) { + return
Loading...
; + } + + const daysLeft = account ? trialDaysLeft(account.trial_ends_at) : 0; + const onTrial = account?.effective_tier === 'ultra' && account?.tier === 'free' && daysLeft > 0; + const badgeTier = account?.effective_tier || account?.tier || 'free'; + const badgeColors = TIER_BADGE[badgeTier] || TIER_BADGE.free; + + return ( +
+ {/* Trial banner */} + {onTrial && ( +
+ Ultra trial ends in {daysLeft} day{daysLeft !== 1 ? 's' : ''} +
+ )} + + {/* Profile */} +
+

Profile

+ {account ? ( +
+
+
Email
+
{account.email}
+
+
+
Plan
+
+ + {onTrial + ? 'FREE (Ultra trial)' + : badgeTier.toUpperCase()} + +
+
+
+
Member since
+
+ {new Date(account.created_at).toLocaleDateString()} +
+
+
+ ) : ( +

Could not load account info.

+ )} + +
+ + {/* Usage */} + {account?.usage && ( +
+

Usage today

+
+ + +
+

+ Resets at {new Date(account.usage.quota_resets_at).toLocaleString()} +

+
+ )} + + {/* API Keys */} +
+

API Keys

+ + {error && ( +
{error}
+ )} + + {createdKey && ( +
+

+ Key created! Copy it now — you won't see it again. +

+ + {createdKey} + +
+ )} + +
+ setNewKeyName(e.target.value)} + className="flex-1 px-3 py-1.5 border border-gray-300 rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-red-500" + /> + +
+ + {apiKeys.length > 0 ? ( +
    + {apiKeys.map((k) => ( +
  • +
    + {k.name} + {k.key_prefix}... +
    + +
    • + ))} +
+ ) : ( +

No API keys yet.

+ )} +
+
+ ); +} + +function UsageBar({ label, used, limit }) { + const isUnlimited = limit === -1; + const isBlocked = limit === 0; + const pct = isUnlimited || isBlocked ? 0 : Math.min((used / limit) * 100, 100); + + return ( +
+
+ {label} + + {isBlocked + ? 'Blocked (upgrade required)' + : isUnlimited + ? `${used} / unlimited` + : `${used} / ${limit}`} + +
+ {!isBlocked && !isUnlimited && ( +
+
= 90 ? 'bg-red-500' : pct >= 70 ? 'bg-yellow-400' : 'bg-green-500' + }`} + style={{ width: `${pct}%` }} + /> +
+ )} +
+ ); +} diff --git a/frontend/src/pages/CookiePage.jsx b/frontend/src/pages/CookiePage.jsx new file mode 100644 index 0000000..bc5cfd3 --- /dev/null +++ b/frontend/src/pages/CookiePage.jsx @@ -0,0 +1,105 @@ +export default function CookiePage() { + return ( +
+

Cookie Policy

+

Last updated: 10 March 2026

+ +
+
+

1. Overview

+

+ Cherry Evals uses minimal browser storage, limited to what is strictly necessary for + the Service to function. We do not use advertising cookies, analytics + trackers, social media pixels, or any third-party tracking scripts. +

+
+ +
+

2. What We Store

+ + + + + + + + + + + + + + + + + + + +
Name / KeyTypePurposeDurationCategory
sb-*-auth-tokenlocalStorage + Stores your Supabase authentication session (JWT access token and refresh token) + so you remain signed in across page loads. + Until sign-out or token expiryStrictly necessary
+

+ That's it. We have no other cookies, local storage entries, + session storage entries, or IndexedDB databases. +

+
+ +
+

3. Third-Party Cookies

+

+ Cherry Evals does not load any third-party scripts that set cookies. We do not use + Google Analytics, Meta Pixel, Hotjar, Intercom, or any other analytics or marketing + tool. The landing page at cherryevals.com is fully static with no external + dependencies. +

+

+ When you sign in via a third-party OAuth provider (e.g., Google, GitHub) through + Supabase, the provider's own authentication flow may set cookies on their domain. + These cookies are governed by the provider's privacy policy, not ours. +

+
+ +
+

4. Why No Cookie Banner?

+

+ Under the ePrivacy Directive (2002/58/EC, as amended by 2009/136/EC) and its national + implementations, consent is not required for storage that is "strictly + necessary" for a service explicitly requested by the user. Our sole use of + localStorage — keeping you signed in — falls squarely within + this exemption. +

+

+ If we ever add non-essential cookies or trackers in the future, we will implement a + consent mechanism before doing so. +

+
+ +
+

5. How to Clear Stored Data

+

You can remove your authentication session at any time by:

+
    +
  • Clicking "Sign out" in the app (this clears the localStorage entry).
    • +
  • Manually clearing your browser's local storage for the app.cherryevals.com domain.
    • +
  • Using your browser's "Clear site data" or "Clear browsing data" feature.
    • +
+
+ +
+

6. Changes

+

+ If we change what we store in your browser, we will update this page. If we introduce + non-essential cookies, we will ask for your consent before setting them. +

+
+ +
+

7. Contact

+

+ Questions about cookies or browser storage: privacy@cherryevals.com +

+
+
+
+ ); +} diff --git a/frontend/src/pages/LoginPage.jsx b/frontend/src/pages/LoginPage.jsx new file mode 100644 index 0000000..7cd66eb --- /dev/null +++ b/frontend/src/pages/LoginPage.jsx @@ -0,0 +1,147 @@ +import { useState } from 'react'; +import { Navigate, useNavigate } from 'react-router-dom'; +import { supabase } from '../lib/supabase'; +import { useAuth } from '../lib/AuthContext'; + +export default function LoginPage() { + const [email, setEmail] = useState(''); + const [password, setPassword] = useState(''); + const [isSignUp, setIsSignUp] = useState(false); + const [error, setError] = useState(''); + const [loading, setLoading] = useState(false); + const navigate = useNavigate(); + const { user } = useAuth(); + + // Already logged in — redirect via component, not imperative navigate in render + if (user) { + return ; + } + + if (!supabase) { + return ( +
+

Authentication not configured

+

+ Supabase is not configured. Set VITE_SUPABASE_URL and VITE_SUPABASE_ANON_KEY to enable auth. +

+
+ ); + } + + const handleEmailAuth = async (e) => { + e.preventDefault(); + setError(''); + setLoading(true); + + try { + const { error: authError } = isSignUp + ? await supabase.auth.signUp({ email, password }) + : await supabase.auth.signInWithPassword({ email, password }); + + if (authError) throw authError; + + if (isSignUp) { + setError('Check your email for a confirmation link.'); + } else { + navigate('/'); + } + } catch (err) { + setError(err.message); + } finally { + setLoading(false); + } + }; + + const handleOAuth = async (provider) => { + setError(''); + const { error: authError } = await supabase.auth.signInWithOAuth({ + provider, + options: { redirectTo: window.location.origin }, + }); + if (authError) setError(authError.message); + }; + + return ( +
+

+ {isSignUp ? 'Create account' : 'Sign in'} +

+ + {error && ( +
{error}
+ )} + +
+
+ + setEmail(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-red-500" + /> +
+
+ + setPassword(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-red-500" + /> +
+ {isSignUp && ( +

+ By creating an account you agree to our{' '} + Terms of Service and{' '} + Privacy Policy. +

+ )} + +
+ +
+
+ or +
+
+ +
+ + +
+ +

+ {isSignUp ? 'Already have an account?' : "Don't have an account?"}{' '} + +

+
+ ); +} diff --git a/frontend/src/pages/PricingPage.jsx b/frontend/src/pages/PricingPage.jsx new file mode 100644 index 0000000..0cdd7d6 --- /dev/null +++ b/frontend/src/pages/PricingPage.jsx @@ -0,0 +1,137 @@ +import { useAuth } from '../lib/AuthContext'; + +const POLAR_PRO_CHECKOUT_URL = import.meta.env.VITE_POLAR_PRO_CHECKOUT_URL || ''; +const POLAR_ULTRA_CHECKOUT_URL = import.meta.env.VITE_POLAR_ULTRA_CHECKOUT_URL || ''; + +const plans = [ + { + name: 'Free', + price: '$0', + period: 'forever', + features: [ + 'Keyword search (30/min)', + '50 semantic/hybrid searches per day', + '10 collections, 1,000 examples each', + 'JSON / JSONL / CSV export', + '1 API key', + 'MCP access (Free limits)', + ], + blocked: [ + 'Intelligent search (LLM-powered)', + 'Agentic ingestion & export', + 'Langfuse export', + ], + cta: 'Current plan', + highlighted: false, + checkoutUrl: null, + }, + { + name: 'Pro', + price: '$19', + period: '/month', + features: [ + 'Keyword search (120/min)', + 'Unlimited semantic/hybrid search', + '180 LLM calls per day (Flash)', + 'Unlimited collections & examples', + 'All export formats incl. Langfuse', + 'Agentic ingestion & export', + '10 API keys', + 'MCP access (Pro limits)', + ], + blocked: [], + cta: 'Upgrade to Pro', + highlighted: false, + checkoutUrl: POLAR_PRO_CHECKOUT_URL, + }, + { + name: 'Ultra', + price: '$49', + period: '/month', + features: [ + 'Everything in Pro', + '300 LLM calls per day (Pro + Flash)', + 'Priority support', + '10 API keys', + ], + blocked: [], + cta: 'Upgrade to Ultra', + highlighted: true, + checkoutUrl: POLAR_ULTRA_CHECKOUT_URL, + }, +]; + +export default function PricingPage() { + const { user } = useAuth(); + + return ( +
+
+

Plans & Pricing

+

+ Free for exploration. Pro for LLM-powered features. Ultra for maximum capacity. +

+

+ Start with a free 1-week Ultra trial +

+
+ +
+ {plans.map((plan) => ( +
+

{plan.name}

+

+ {plan.price} + {plan.period} +

+ +
    + {plan.features.map((f) => ( +
  • + ✓ + {f} +
    • + ))} + {plan.blocked.map((f) => ( +
  • + ✗ + {f} +
    • + ))} +
+ +
+ {plan.checkoutUrl ? ( + + {plan.cta} + + ) : plan.name !== 'Free' ? ( + + Coming soon + + ) : ( + + {user ? plan.cta : 'Sign up free'} + + )} +
+
+ ))} +
+
+ ); +} diff --git a/frontend/src/pages/PrivacyPage.jsx b/frontend/src/pages/PrivacyPage.jsx new file mode 100644 index 0000000..3458f88 --- /dev/null +++ b/frontend/src/pages/PrivacyPage.jsx @@ -0,0 +1,220 @@ +export default function PrivacyPage() { + return ( +
+

Privacy Policy

+

Last updated: 10 March 2026

+ +
+
+

1. Controller

+

+ Cherry Evals ("we", "us", "the Service") is operated by + Emilio Marinone, a sole proprietor based in the European Union. + For data-protection enquiries: privacy@cherryevals.com. +

+
+ +
+

2. What We Collect and Why

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DataPurposeLegal Basis (GDPR)
Email addressAccount identification, billing, service communicationsPerformance of contract (Art. 6(1)(b))
Supabase user IDAuthenticate requests, link data to accountPerformance of contract (Art. 6(1)(b))
API key hash & prefixProgrammatic authentication (raw key never stored)Performance of contract (Art. 6(1)(b))
Daily usage countersQuota enforcement per subscription tierPerformance of contract (Art. 6(1)(b))
Search queries & curation events (pick, remove, export)Service functionality; aggregated to improve search relevance for all users ("collective intelligence")Legitimate interest (Art. 6(1)(f)) — you may object (see Section 7)
IP address (transient)Rate limiting; not persisted to databaseLegitimate interest (Art. 6(1)(f))
Subscription & payment metadata (via Polar.sh)Tier assignment, billing reconciliationPerformance of contract (Art. 6(1)(b))
+

+ We do not collect passwords (handled entirely by Supabase), + payment card details (handled entirely by Polar.sh/Stripe), or biometric data. +

+
+ +
+

3. AI-Powered Features

+

+ When you use "Intelligent Search", "Agentic Ingestion", or "Custom Export", + your search query or format description is sent to third-party large language model (LLM) + providers (currently Google Gemini and Anthropic Claude) for processing. These providers + act as sub-processors and process data under their applicable data processing agreements. +

+

+ The LLMs are used only to interpret your search intent, rank results, + generate dataset parsers, or produce export format converters. No automated decisions + are made that produce legal or similarly significant effects on you. You always review + and approve AI-generated results before they are applied. +

+
+ +
+

4. Sub-processors

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ProviderRoleLocation
Supabase Inc.Authentication (JWT, OAuth)US (EU project region available)
Polar.shSubscription billing, webhooksEU/US
Google (Gemini API)Embeddings, search intelligence, ingestion, exportUS
Anthropic (Claude API)Advanced reasoning tasksUS
QdrantVector database (semantic search)Self-hosted or Qdrant Cloud (EU/US)
LangfuseObservability traces (dataset content only, no user PII)EU
+

+ Transfers to US-based sub-processors rely on EU Standard Contractual Clauses (SCCs) + or the EU-US Data Privacy Framework where applicable. We maintain Data Processing + Agreements (DPAs) with each sub-processor. +

+
+ +
+

5. Data Retention

+
    +
  • Account data: retained while your account is active. Deleted within 30 days of account deletion request.
    • +
  • Curation events: retained for 24 months for collective intelligence purposes, then anonymised (user_id removed) or deleted.
    • +
  • API key metadata: deleted when you revoke the key or delete your account.
    • +
  • Usage counters: reset daily; historical counts are not retained.
    • +
  • IP addresses: held in memory only for rate limiting; never persisted to disk.
    • +
+
+ +
+

6. Cookies & Local Storage

+

+ We use browser localStorage to store your authentication session token + (provided by Supabase). This is a strictly necessary functional mechanism and does not + require consent under the ePrivacy Directive. We do not use advertising cookies, + analytics trackers, or third-party tracking scripts. See our{' '} + Cookie Policy for details. +

+
+ +
+

7. Your Rights (GDPR)

+

You have the right to:

+
    +
  • Access — request a copy of all personal data we hold about you.
    • +
  • Rectification — correct inaccurate data.
    • +
  • Erasure — request deletion of your account and all associated data ("right to be forgotten").
    • +
  • Data portability — receive your data in a structured, machine-readable format (JSON).
    • +
  • Restriction — request that we limit processing of your data.
    • +
  • Object — object to processing based on legitimate interest (e.g., curation event collection for collective intelligence).
    • +
  • Withdraw consent — where processing is based on consent, withdraw at any time.
    • +
+

+ To exercise any right, email privacy@cherryevals.com. We will respond + within 30 days. You also have the right to lodge a complaint with your local data + protection authority. For users in Sweden, this is{' '} + Integritetsskyddsmyndigheten (IMY) —{' '} + www.imy.se. +

+
+ +
+

8. Mandatory Data Provision

+

+ Providing your email address is required to create an account. If you choose not to + provide it, you may still use the Service without an account (keyword, semantic, and + hybrid search only). LLM-powered features, collections, and exports require an account. +

+
+ +
+

9. Security

+

+ We protect your data with: HTTPS encryption in transit, bcrypt/SHA-256 hashing for + credentials, role-based access controls, rate limiting, and daily quota enforcement. + API keys are stored as SHA-256 hashes; the raw key is shown once at creation and never + stored. Despite these measures, no system is 100 % secure. If you discover a + vulnerability, please report it to security@cherryevals.com. +

+
+ +
+

10. Children

+

+ Cherry Evals is not directed at individuals under 16. We do not knowingly collect data + from children. If you believe a child has provided us with personal data, contact us and + we will delete it. +

+
+ +
+

11. Changes

+

+ We may update this policy. Material changes will be communicated via email or an in-app + notice at least 14 days before they take effect. Continued use after the effective date + constitutes acceptance of the updated policy. +

+
+
+
+ ); +} diff --git a/frontend/src/pages/TermsPage.jsx b/frontend/src/pages/TermsPage.jsx new file mode 100644 index 0000000..bd005a2 --- /dev/null +++ b/frontend/src/pages/TermsPage.jsx @@ -0,0 +1,272 @@ +export default function TermsPage() { + return ( +
+

Terms of Service

+

Last updated: 10 March 2026

+ +
+
+

1. Agreement

+

+ By accessing or using Cherry Evals ("the Service"), operated by Emilio Marinone + ("we", "us"), you agree to these Terms of Service ("Terms"). + If you do not agree, do not use the Service. If you are using the Service on behalf of + an organisation, you represent that you have authority to bind that organisation. +

+
+ +
+

2. Service Description

+

+ Cherry Evals is a platform for discovering, searching, curating, and exporting + examples from public AI evaluation benchmark datasets. The Service is available + via a web interface, REST API, CLI, and MCP server. +

+

+ Certain features ("Intelligent Search", "Agentic Ingestion", + "Custom Export") use large language models (LLMs) to assist with query + interpretation, result ranking, dataset parsing, and format conversion. These features + are clearly labelled and subject to per-tier usage quotas. +

+
+ +
+

3. Accounts and API Keys

+
    +
  • You must provide a valid email address to create an account.
    • +
  • You are responsible for all activity under your account and API keys.
    • +
  • Do not share API keys. Treat them as passwords.
    • +
  • Notify us immediately at security@cherryevals.com if you suspect unauthorised use.
    • +
  • We may suspend or terminate accounts that violate these Terms.
    • +
+
+ +
+

4. Acceptable Use

+

You may use the Service only for:

+
    +
  • Searching and browsing public AI evaluation datasets.
    • +
  • Creating, managing, and exporting curated collections of evaluation examples.
    • +
  • Using LLM-powered features (intelligent search, agentic ingestion, custom export) for the above purposes.
    • +
  • Integrating via the API, CLI, or MCP server for the above purposes.
    • +
+
+ +
+

5. Prohibited Uses

+

You must not:

+
    +
  • + Abuse LLM features — use intelligent search, agentic ingestion, + or custom export to generate answers to evaluation questions, use the LLM as a + general-purpose assistant, extract training data, or otherwise use LLM credits for + purposes outside the product scope. +
    • +
  • + Attempt prompt injection — craft search queries, format + descriptions, or dataset descriptions designed to manipulate the behaviour of the + underlying LLMs, bypass safety controls, or extract system prompts. +
    • +
  • + Circumvent quotas or rate limits — create multiple accounts, + rotate API keys, or use technical means to exceed your subscription tier limits. +
    • +
  • + Scrape or bulk-download — systematically extract data beyond + what is necessary for legitimate evaluation curation. +
    • +
  • + Reverse-engineer — decompile, disassemble, or reverse-engineer + any part of the Service (except as permitted by applicable law). +
    • +
  • + Interfere with the Service — introduce malware, overload + infrastructure, exploit vulnerabilities, or access other users' data without + authorisation. +
    • +
  • + Ingest malicious content — submit datasets or descriptions + containing prompt injection payloads, executable code, or content designed to + compromise the Service or other users. +
    • +
  • + Violate third-party rights — use the Service in a way that + infringes intellectual property, privacy, or other rights of third parties. +
    • +
  • + Use the Service for illegal purposes — violate any applicable + law or regulation. +
    • +
+

+ Violation of these restrictions may result in immediate suspension or termination of + your account without notice. +

+
+ +
+

6. Subscription Tiers and Billing

+
    +
  • + Free tier: keyword, semantic, and hybrid search; collection management; + standard export formats. No LLM-powered features. +
    • +
  • + Pro ($19/month) and Ultra ($49/month): include + intelligent search, agentic ingestion, and custom export, subject to daily quotas. +
    • +
  • New accounts receive a 7-day Ultra trial.
    • +
  • Billing is handled by Polar.sh. By subscribing, you also agree to Polar.sh's terms of service.
    • +
  • Subscriptions renew automatically. Cancel at any time; access continues until the end of the billing period.
    • +
  • We reserve the right to change pricing with 30 days' notice.
    • +
+
+ +
+

7. Intellectual Property

+
    +
  • + Service: Cherry Evals is licensed under the Elastic License 2.0. + You may use and modify the source code, but you may not provide the Service as a + competing hosted offering. +
    • +
  • + Datasets: evaluation datasets accessible through the Service are + third-party works. You are responsible for complying with each dataset's licence + (typically CC-BY, MIT, or Apache 2.0). We do not claim ownership of these datasets. +
    • +
  • + Your collections: you retain ownership of the collections you create. + By using the Service, you grant us a limited licence to process your collections for + service delivery and to use anonymised, aggregated curation signals to improve the + Service for all users. +
    • +
+
+ +
+

8. AI Transparency (EU AI Act)

+

+ In compliance with Article 50 of the EU AI Act, we disclose the following: +

+
    +
  • Features labelled "Intelligent", "Agentic", or "AI-powered" use large language models (Google Gemini, Anthropic Claude) to process your input.
    • +
  • AI-generated results (search rankings, dataset parsers, export converters) are always presented for your review before being applied.
    • +
  • No fully automated decisions with legal or significant effects are made by the Service.
    • +
  • AI operations are logged for auditability. You may request access to logs related to your usage.
    • +
+
+ +
+

9. Disclaimer of Warranties

+

+ The Service is provided "as is" and "as available". To the maximum + extent permitted by law, we disclaim all warranties, express or implied, including + warranties of merchantability, fitness for a particular purpose, and non-infringement. +

+

+ We do not warrant that: (a) the Service will be uninterrupted or error-free; + (b) AI-generated results will be accurate, complete, or suitable for your purpose; + (c) evaluation datasets are free of errors or biases. +

+
+ +
+

10. Limitation of Liability

+

+ To the maximum extent permitted by law, our total liability to you for all claims + arising from or related to the Service shall not exceed the amount you paid us in the + 12 months preceding the claim. We are not liable for indirect, incidental, special, + consequential, or punitive damages, including lost profits, data loss, or business + interruption. +

+
+ +
+

11. Termination

+
    +
  • You may delete your account at any time (see Privacy Policy for data deletion details).
    • +
  • We may suspend or terminate your account for violation of these Terms, with or without notice depending on severity.
    • +
  • Upon termination, your right to use the Service ceases immediately. We will delete your data in accordance with our Privacy Policy.
    • +
+
+ +
+

12. Right of Withdrawal (EU Consumers)

+

+ If you are a consumer in the EU, you have a 14-day right of withdrawal from the date + of purchase. However, by using the Service immediately after subscribing, you consent + to the commencement of service delivery before the withdrawal period expires. Once you + have used LLM-powered features, the right of withdrawal is waived to the extent of + services already provided. To exercise the right of withdrawal, email{' '} + legal@cherryevals.com within 14 days of purchase. +

+
+ +
+

13. Force Majeure

+

+ We are not liable for failure or delay in performing obligations due to events beyond + our reasonable control, including but not limited to: natural disasters, pandemics, + third-party service outages (Supabase, Polar.sh, Google, Anthropic), government actions, + or internet infrastructure failures. +

+
+ +
+

14. Governing Law

+

+ These Terms are governed by the laws of Sweden. Disputes shall be resolved in the + courts of Stockholm, Sweden. If you are a consumer in the EU, you retain the protection + of mandatory provisions of the law of your country of residence, and you may bring + proceedings in the courts of your country of residence. +

+
+ +
+

15. Severability

+

+ If any provision of these Terms is found to be invalid or unenforceable, the remaining + provisions remain in full force and effect. The invalid provision shall be replaced by + a valid provision that most closely achieves the intended economic purpose. +

+
+ +
+

16. Entire Agreement

+

+ These Terms, together with the Privacy Policy and Cookie Policy, constitute the entire + agreement between you and Cherry Evals regarding the Service. They supersede all prior + oral or written communications. +

+
+ +
+

17. Assignment

+

+ You may not assign your rights or obligations under these Terms without our prior + written consent. We may assign our rights and obligations to a successor in connection + with a merger, acquisition, or sale of all or substantially all of our assets. +

+
+ +
+

18. Changes

+

+ We may update these Terms. Material changes will be communicated via email or in-app + notice at least 30 days before they take effect. Continued use after the effective + date constitutes acceptance. If you disagree, you must stop using the Service and + delete your account. +

+
+ +
+

19. Contact

+

+ Questions about these Terms: legal@cherryevals.com +

+
+
+
+ ); +} diff --git a/landing/index.html b/landing/index.html index c64c9e2..7522a71 100644 --- a/landing/index.html +++ b/landing/index.html @@ -439,6 +439,12 @@

Or self-host it.

+
+ Terms + Privacy + Cookies +
+

Elastic License 2.0 — Free to use and modify

diff --git a/pyproject.toml b/pyproject.toml index fd7608c..b2ffc0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "psycopg2-binary>=2.9.11", "pydantic>=2.12.5", "pydantic-settings>=2.12.0", + "pyjwt>=2.11.0", "python-dotenv>=1.2.1", "qdrant-client>=1.16.2", "sqlalchemy>=2.0.45", diff --git a/tests/conftest.py b/tests/conftest.py index 46a1c26..7c1f726 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,16 +1,20 @@ """Shared test fixtures for all test types.""" import os +from contextlib import contextmanager from pathlib import Path +from unittest.mock import patch import pytest from fastapi.testclient import TestClient from sqlalchemy import create_engine, event from sqlalchemy.orm import sessionmaker +from api.deps import get_current_user, get_optional_user from api.main import app from cherry_evals.config import Settings from db.postgres.base import Base, get_db +from db.postgres.models import User @pytest.fixture(scope="session") @@ -21,6 +25,7 @@ def test_settings(): qdrant_url="http://localhost:6333", google_api_key="test-api-key", cherry_data_dir=Path("./test_data"), + auth_enabled=False, ) @@ -66,8 +71,8 @@ def end_savepoint(session, transaction): @pytest.fixture(scope="function") -def test_client(test_db_session): - """Create a FastAPI test client with test database.""" +def test_client(test_db_session, test_settings): + """Create a FastAPI test client with test database and auth disabled.""" def override_get_db(): try: @@ -77,12 +82,168 @@ def override_get_db(): app.dependency_overrides[get_db] = override_get_db - with TestClient(app) as client: + with ( + patch("api.deps.settings", test_settings), + patch("api.routes.export.settings", test_settings), + patch("api.routes.collections.settings", test_settings), + TestClient(app) as client, + ): yield client app.dependency_overrides.clear() +# --------------------------------------------------------------------------- +# Auth test helpers +# --------------------------------------------------------------------------- + + +def _make_fake_user( + *, tier: str = "free", supabase_id: str = "test-user-123", trial_ends_at=None +) -> User: + """Build an in-memory User object for dependency overrides.""" + from datetime import UTC, datetime, timedelta + + return User( + id=1, + supabase_id=supabase_id, + email=f"{tier}@test.com", + tier=tier, + trial_ends_at=trial_ends_at, + polar_customer_id=None, + polar_subscription_id=None, + subscription_status=None, + llm_calls_today=0, + semantic_searches_today=0, + quota_reset_at=datetime.now(UTC) + timedelta(days=1), + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + ) + + +@pytest.fixture +def free_user(): + """A Free-tier User object.""" + return _make_fake_user(tier="free") + + +@pytest.fixture +def pro_user(): + """A Pro-tier User object.""" + return _make_fake_user(tier="pro", supabase_id="pro-user-456") + + +@pytest.fixture +def ultra_user(): + """An Ultra-tier User object.""" + return _make_fake_user(tier="ultra", supabase_id="ultra-user-789") + + +@pytest.fixture +def trial_user(): + """A Free-tier user with an active Ultra trial.""" + from datetime import UTC, datetime, timedelta + + return _make_fake_user( + tier="free", + supabase_id="trial-user-101", + trial_ends_at=datetime.now(UTC) + timedelta(days=7), + ) + + +@pytest.fixture +def expired_trial_user(): + """A Free-tier user with an expired trial.""" + from datetime import UTC, datetime, timedelta + + return _make_fake_user( + tier="free", + supabase_id="expired-trial-user-102", + trial_ends_at=datetime.now(UTC) - timedelta(days=1), + ) + + +# --------------------------------------------------------------------------- +# Authenticated test client factory +# --------------------------------------------------------------------------- + + +def _make_authed_client(test_db_session, user): + """Create a test client authenticated as the given user. + + Patches settings across all route modules that reference them directly. + """ + authed_settings = Settings( + database_url="sqlite:///./test.db", + qdrant_url="http://localhost:6333", + google_api_key="test-api-key", + cherry_data_dir=Path("./test_data"), + auth_enabled=True, + supabase_jwt_secret="test-secret", + ) + + def override_get_db(): + try: + yield test_db_session + finally: + pass + + def override_user(): + return user + + app.dependency_overrides[get_db] = override_get_db + app.dependency_overrides[get_current_user] = override_user + app.dependency_overrides[get_optional_user] = override_user + + @contextmanager + def _client_context(): + with ( + patch("api.deps.settings", authed_settings), + patch("api.routes.export.settings", authed_settings), + patch("api.routes.collections.settings", authed_settings), + TestClient(app) as client, + ): + yield client + app.dependency_overrides.clear() + + return _client_context() + + +@pytest.fixture +def authed_client_free(test_db_session, free_user): + """Test client authenticated as a Free user (auth_enabled=True).""" + with _make_authed_client(test_db_session, free_user) as client: + yield client + + +@pytest.fixture +def authed_client_pro(test_db_session, pro_user): + """Test client authenticated as a Pro user (auth_enabled=True).""" + with _make_authed_client(test_db_session, pro_user) as client: + yield client + + +@pytest.fixture +def authed_client_ultra(test_db_session, ultra_user): + """Test client authenticated as an Ultra user (auth_enabled=True).""" + with _make_authed_client(test_db_session, ultra_user) as client: + yield client + + +@pytest.fixture +def authed_client_trial(test_db_session, trial_user): + """Test client authenticated as a Free user with active Ultra trial.""" + with _make_authed_client(test_db_session, trial_user) as client: + yield client + + +@pytest.fixture +def authed_client_expired_trial(test_db_session, expired_trial_user): + """Test client authenticated as a Free user with expired trial.""" + with _make_authed_client(test_db_session, expired_trial_user) as client: + yield client + + @pytest.fixture(scope="session", autouse=True) def set_test_env(): """Set test environment variables.""" diff --git a/tests/integration/test_auth.py b/tests/integration/test_auth.py new file mode 100644 index 0000000..03aacdb --- /dev/null +++ b/tests/integration/test_auth.py @@ -0,0 +1,293 @@ +"""Auth, authorization, and tier enforcement tests.""" + +from pathlib import Path +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from api.main import app +from cherry_evals.config import Settings +from db.postgres.base import get_db + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_AUTH_ENABLED_SETTINGS = Settings( + database_url="sqlite:///./test.db", + qdrant_url="http://localhost:6333", + google_api_key="test-api-key", + cherry_data_dir=Path("./test_data"), + auth_enabled=True, + supabase_jwt_secret="test-secret", +) + + +@pytest.fixture +def unauthed_client(test_db_session): + """Test client with auth_enabled=True but NO credentials.""" + + def override_get_db(): + try: + yield test_db_session + finally: + pass + + app.dependency_overrides[get_db] = override_get_db + # Don't override get_current_user — let it run naturally + + with ( + patch("api.deps.settings", _AUTH_ENABLED_SETTINGS), + patch("api.routes.export.settings", _AUTH_ENABLED_SETTINGS), + patch("api.routes.collections.settings", _AUTH_ENABLED_SETTINGS), + TestClient(app) as client, + ): + yield client + + app.dependency_overrides.clear() + + +# --------------------------------------------------------------------------- +# 401 on protected endpoints without auth +# --------------------------------------------------------------------------- + + +class TestUnauthenticatedAccess: + """Endpoints that require auth should return 401 when no credentials.""" + + def test_create_collection_requires_auth(self, unauthed_client): + resp = unauthed_client.post("/collections", json={"name": "test"}) + assert resp.status_code == 401 + + def test_list_collections_requires_auth(self, unauthed_client): + resp = unauthed_client.get("/collections") + assert resp.status_code == 401 + + def test_create_api_key_requires_auth(self, unauthed_client): + resp = unauthed_client.post("/api-keys", json={"name": "test"}) + assert resp.status_code == 401 + + def test_list_api_keys_requires_auth(self, unauthed_client): + resp = unauthed_client.get("/api-keys") + assert resp.status_code == 401 + + def test_account_me_requires_auth(self, unauthed_client): + resp = unauthed_client.get("/account/me") + assert resp.status_code == 401 + + +# --------------------------------------------------------------------------- +# 403 on Pro endpoints with Free user +# --------------------------------------------------------------------------- + + +class TestFreeTierRestrictions: + """Pro-only endpoints should return 403 for Free-tier users.""" + + def test_intelligent_search_blocked_for_free(self, authed_client_free): + resp = authed_client_free.post( + "/search/intelligent", + json={"query": "test", "limit": 5}, + ) + assert resp.status_code == 403 + + def test_agent_discover_blocked_for_free(self, authed_client_free): + resp = authed_client_free.post( + "/agents/discover", + json={"description": "test"}, + ) + assert resp.status_code == 403 + + def test_agent_ingest_blocked_for_free(self, authed_client_free): + resp = authed_client_free.post( + "/agents/ingest", + json={"description": "test"}, + ) + assert resp.status_code == 403 + + +# --------------------------------------------------------------------------- +# Authenticated Free user can access basic endpoints +# --------------------------------------------------------------------------- + + +class TestFreeTierAccess: + """Free users should be able to use basic endpoints.""" + + def test_free_user_can_create_collection(self, authed_client_free): + resp = authed_client_free.post( + "/collections", + json={"name": "My test collection"}, + ) + assert resp.status_code == 201 + data = resp.json() + assert data["name"] == "My test collection" + + def test_free_user_can_list_collections(self, authed_client_free): + resp = authed_client_free.get("/collections") + assert resp.status_code == 200 + + def test_free_user_can_keyword_search(self, authed_client_free): + resp = authed_client_free.post( + "/search", + json={"query": "test"}, + ) + assert resp.status_code == 200 + + def test_free_user_can_get_account(self, authed_client_free): + resp = authed_client_free.get("/account/me") + assert resp.status_code == 200 + data = resp.json() + assert data["tier"] == "free" + assert data["effective_tier"] == "free" + assert data["email"] == "free@test.com" + + +# --------------------------------------------------------------------------- +# Public endpoints stay public +# --------------------------------------------------------------------------- + + +class TestPublicEndpoints: + """Endpoints that should work without auth (health, facets, datasets).""" + + def test_health_no_auth(self, unauthed_client): + resp = unauthed_client.get("/health") + assert resp.status_code == 200 + + def test_root_no_auth(self, unauthed_client): + resp = unauthed_client.get("/") + assert resp.status_code == 200 + + def test_facets_no_auth(self, unauthed_client): + """Facets endpoint doesn't require auth.""" + resp = unauthed_client.post("/search/facets", json={}) + assert resp.status_code == 200 + + +# --------------------------------------------------------------------------- +# Existing tests still pass with auth_enabled=False +# --------------------------------------------------------------------------- + + +class TestAuthDisabledMode: + """When auth_enabled=False, all endpoints work without credentials.""" + + def test_collections_work_without_auth(self, test_client): + resp = test_client.post("/collections", json={"name": "no-auth collection"}) + assert resp.status_code == 201 + + def test_search_works_without_auth(self, test_client): + resp = test_client.post("/search", json={"query": "test"}) + assert resp.status_code == 200 + + +# --------------------------------------------------------------------------- +# Polar webhook +# --------------------------------------------------------------------------- + + +class TestUltraTierAccess: + """Ultra-tier users should access all paid endpoints.""" + + def test_ultra_user_can_access_intelligent_search(self, authed_client_ultra): + """Ultra user gets past the require_paid gate (may fail on LLM, but not 403).""" + resp = authed_client_ultra.post( + "/search/intelligent", + json={"query": "test", "limit": 5}, + ) + # Should not be 403 (tier restriction) + assert resp.status_code != 403 + + def test_ultra_user_can_access_agents(self, authed_client_ultra): + resp = authed_client_ultra.post( + "/agents/discover", + json={"description": "test"}, + ) + assert resp.status_code != 403 + + def test_ultra_user_account_shows_ultra(self, authed_client_ultra): + resp = authed_client_ultra.get("/account/me") + assert resp.status_code == 200 + data = resp.json() + assert data["tier"] == "ultra" + assert data["effective_tier"] == "ultra" + + +# --------------------------------------------------------------------------- +# Trial user (free + active trial_ends_at) gets Ultra access +# --------------------------------------------------------------------------- + + +class TestTrialAccess: + """Free users with active trial should get Ultra-level access.""" + + def test_trial_user_can_access_intelligent_search(self, authed_client_trial): + resp = authed_client_trial.post( + "/search/intelligent", + json={"query": "test", "limit": 5}, + ) + assert resp.status_code != 403 + + def test_trial_user_can_access_agents(self, authed_client_trial): + resp = authed_client_trial.post( + "/agents/discover", + json={"description": "test"}, + ) + assert resp.status_code != 403 + + def test_trial_user_account_shows_effective_ultra(self, authed_client_trial): + resp = authed_client_trial.get("/account/me") + assert resp.status_code == 200 + data = resp.json() + assert data["tier"] == "free" + assert data["effective_tier"] == "ultra" + assert data["trial_ends_at"] is not None + + +# --------------------------------------------------------------------------- +# Expired trial user falls back to Free limits +# --------------------------------------------------------------------------- + + +class TestExpiredTrialRestrictions: + """Free users with expired trial should be blocked from paid features.""" + + def test_expired_trial_blocked_from_intelligent_search(self, authed_client_expired_trial): + resp = authed_client_expired_trial.post( + "/search/intelligent", + json={"query": "test", "limit": 5}, + ) + assert resp.status_code == 403 + + def test_expired_trial_blocked_from_agents(self, authed_client_expired_trial): + resp = authed_client_expired_trial.post( + "/agents/discover", + json={"description": "test"}, + ) + assert resp.status_code == 403 + + def test_expired_trial_account_shows_free(self, authed_client_expired_trial): + resp = authed_client_expired_trial.get("/account/me") + assert resp.status_code == 200 + data = resp.json() + assert data["tier"] == "free" + assert data["effective_tier"] == "free" + + +# --------------------------------------------------------------------------- +# Polar webhook +# --------------------------------------------------------------------------- + + +class TestPolarWebhook: + """Polar webhook signature verification and tier changes.""" + + def test_webhook_rejects_bad_signature(self, unauthed_client): + resp = unauthed_client.post( + "/webhooks/polar", + json={"type": "subscription.created", "data": {}}, + headers={"webhook-signature": "bad-signature"}, + ) + assert resp.status_code == 400 diff --git a/tests/integration/test_export_api.py b/tests/integration/test_export_api.py index c48040d..0568ca7 100644 --- a/tests/integration/test_export_api.py +++ b/tests/integration/test_export_api.py @@ -208,4 +208,4 @@ def test_langfuse_without_credentials_returns_502( ) assert response.status_code == 502 - assert "credentials" in response.json()["detail"].lower() + assert "langfuse" in response.json()["detail"].lower() diff --git a/tests/integration/test_rate_limits.py b/tests/integration/test_rate_limits.py new file mode 100644 index 0000000..ff2f9df --- /dev/null +++ b/tests/integration/test_rate_limits.py @@ -0,0 +1,95 @@ +"""Rate limiting and quota enforcement tests.""" + +from datetime import UTC, datetime, timedelta + +import pytest + +from api.deps import FREE_LIMITS, PRO_LIMITS, ULTRA_LIMITS, _rate_limit_buckets, effective_tier +from tests.conftest import _make_fake_user + + +@pytest.fixture(autouse=True) +def clear_rate_limit_state(): + """Reset in-memory rate limit buckets between tests.""" + _rate_limit_buckets.clear() + yield + _rate_limit_buckets.clear() + + +class TestKeywordSearchRateLimit: + """Per-minute rate limiting on keyword search.""" + + def test_keyword_search_within_limit(self, authed_client_free): + """Free user can search within the rate limit.""" + resp = authed_client_free.post("/search", json={"query": "test"}) + assert resp.status_code == 200 + + +class TestSemanticSearchQuota: + """Daily semantic search quota enforcement.""" + + def test_semantic_search_quota_not_enforced_when_disabled(self, test_client): + """Auth disabled: no quota enforcement.""" + # This might fail due to Qdrant not being available, which is expected + # We just verify it doesn't fail with 429 (quota exceeded) + resp = test_client.post( + "/search/semantic", + json={"query": "test", "limit": 5}, + ) + assert resp.status_code != 429 + + +class TestTierLimits: + """Verify tier limit constants are sensible.""" + + def test_free_limits_are_restrictive(self): + assert FREE_LIMITS["llm_calls_per_day"] == 0 + assert FREE_LIMITS["semantic_searches_per_day"] == 50 + assert FREE_LIMITS["max_collections"] == 10 + assert FREE_LIMITS["max_api_keys"] == 1 + + def test_pro_limits_are_generous(self): + assert PRO_LIMITS["llm_calls_per_day"] == 180 + assert PRO_LIMITS["semantic_searches_per_day"] == -1 # unlimited + assert PRO_LIMITS["max_collections"] == -1 # unlimited + assert PRO_LIMITS["max_api_keys"] == 10 + + def test_ultra_limits(self): + assert ULTRA_LIMITS["llm_calls_per_day"] == 300 + assert ULTRA_LIMITS["semantic_searches_per_day"] == -1 # unlimited + assert ULTRA_LIMITS["max_collections"] == -1 # unlimited + assert ULTRA_LIMITS["max_api_keys"] == 10 + + def test_pro_rate_limit_higher_than_free(self): + assert PRO_LIMITS["keyword_rpm"] > FREE_LIMITS["keyword_rpm"] + + def test_ultra_llm_limit_higher_than_pro(self): + assert ULTRA_LIMITS["llm_calls_per_day"] > PRO_LIMITS["llm_calls_per_day"] + + +class TestEffectiveTier: + """Test effective_tier() logic.""" + + def test_free_user_no_trial(self): + user = _make_fake_user(tier="free") + assert effective_tier(user) == "free" + + def test_free_user_active_trial(self): + user = _make_fake_user(tier="free", trial_ends_at=datetime.now(UTC) + timedelta(days=7)) + assert effective_tier(user) == "ultra" + + def test_free_user_expired_trial(self): + user = _make_fake_user(tier="free", trial_ends_at=datetime.now(UTC) - timedelta(days=1)) + assert effective_tier(user) == "free" + + def test_pro_user_ignores_trial(self): + user = _make_fake_user( + tier="pro", + supabase_id="pro-1", + trial_ends_at=datetime.now(UTC) + timedelta(days=7), + ) + assert effective_tier(user) == "pro" + + def test_ultra_user(self): + user = _make_fake_user(tier="ultra", supabase_id="ultra-1") + assert effective_tier(user) == "ultra" diff --git a/uv.lock b/uv.lock index 55809a7..2131034 100644 --- a/uv.lock +++ b/uv.lock @@ -276,6 +276,7 @@ dependencies = [ { name = "psycopg2-binary" }, { name = "pydantic" }, { name = "pydantic-settings" }, + { name = "pyjwt" }, { name = "python-dotenv" }, { name = "qdrant-client" }, { name = "sqlalchemy" }, @@ -304,6 +305,7 @@ requires-dist = [ { name = "psycopg2-binary", specifier = ">=2.9.11" }, { name = "pydantic", specifier = ">=2.12.5" }, { name = "pydantic-settings", specifier = ">=2.12.0" }, + { name = "pyjwt", specifier = ">=2.11.0" }, { name = "python-dotenv", specifier = ">=1.2.1" }, { name = "qdrant-client", specifier = ">=1.16.2" }, { name = "sqlalchemy", specifier = ">=2.0.45" },